From 143cae25da50e978cff03053c9e0f3ce1f7797af Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 17:15:40 +1100 Subject: [PATCH] Implement filtering of github entities based on presence This implements the config from the previous commit. The getRepositories graphql query is updated to include information about the specified catalog info file. When validation is turned on, if the file is not present or empty, the location will not be emitted. Signed-off-by: Nikolas Skoufis --- .../src/lib/github.test.ts | 18 ++- .../src/lib/github.ts | 17 ++- .../providers/GitHubEntityProvider.test.ts | 134 ++++++++++++++++++ .../src/providers/GitHubEntityProvider.ts | 11 ++ 4 files changed, 175 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index e2303c16e6..25878c2005 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -213,6 +213,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -222,6 +223,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], pageInfo: { @@ -243,6 +249,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -252,6 +259,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], }; @@ -262,9 +274,9 @@ describe('github', () => { ), ); - await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( - output, - ); + await expect( + getOrganizationRepositories(graphql, 'a', 'catalog-info.yaml'), + ).resolves.toEqual(output); }); }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 6a944af330..d854cc9043 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -65,6 +65,11 @@ export type Repository = { defaultBranchRef: { name: string; } | null; + catalogInfoFile: { + __typename: string; + id: string; + text: string; + } | null; }; type RepositoryTopics = { @@ -266,14 +271,22 @@ export async function getOrganizationTeams( export async function getOrganizationRepositories( client: typeof graphql, org: string, + catalogPath: string, ): Promise<{ repositories: Repository[] }> { + const catalogPathRef = `HEAD:${catalogPath}`; const query = ` - query repositories($org: String!, $cursor: String) { + query repositories($org: String!, $catalogPathRef: String!, $cursor: String) { repositoryOwner(login: $org) { login repositories(first: 100, after: $cursor) { nodes { name + catalogInfoFile: object(expression: $catalogPathRef) { + __typename + ... on Blob { + id + } + } url isArchived repositoryTopics(first: 100) { @@ -302,7 +315,7 @@ export async function getOrganizationRepositories( query, r => r.repositoryOwner?.repositories, x => x, - { org }, + { org, catalogPathRef }, ); return { repositories }; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 63fb77288b..8b315a8a1e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -170,6 +170,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -267,6 +272,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -341,6 +351,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -381,6 +396,110 @@ describe('GitHubEntityProvider', () => { entities: expectedEntities, }); }); + + it('should filter out invalid locations when validateLocationsExist is set to true', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'catalog-custom.yaml', + filters: { + branch: 'main', + }, + validateLocationsExist: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: null, + }, + { + name: 'another-repo', + url: 'https://github.com/test-org/another-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/another-repo/blob/main/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-934f500db2ba2e8ea3524567926f45a73bb0b532', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); }); it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { @@ -437,6 +556,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-2', @@ -455,6 +579,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-3', @@ -470,6 +599,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3909b5a3c2..42dd6afa5d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -163,6 +163,7 @@ export class GitHubEntityProvider implements EntityProvider { private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; + const catalogPath = this.config.catalogPath; const orgUrl = `https://${host}/${organization}`; const { headers } = await this.githubCredentialsProvider.getCredentials({ @@ -177,8 +178,18 @@ export class GitHubEntityProvider implements EntityProvider { const { repositories } = await getOrganizationRepositories( client, organization, + catalogPath, ); + if (this.config.validateLocationsExist) { + return repositories.filter(repository => { + return ( + repository.catalogInfoFile?.__typename === 'Blob' && + repository.catalogInfoFile.text !== '' + ); + }); + } + return repositories; }