diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index e2303c16e6..25878c2005 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -213,6 +213,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -222,6 +223,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], pageInfo: { @@ -243,6 +249,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -252,6 +259,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], }; @@ -262,9 +274,9 @@ describe('github', () => { ), ); - await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( - output, - ); + await expect( + getOrganizationRepositories(graphql, 'a', 'catalog-info.yaml'), + ).resolves.toEqual(output); }); }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 6a944af330..d854cc9043 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -65,6 +65,11 @@ export type Repository = { defaultBranchRef: { name: string; } | null; + catalogInfoFile: { + __typename: string; + id: string; + text: string; + } | null; }; type RepositoryTopics = { @@ -266,14 +271,22 @@ export async function getOrganizationTeams( export async function getOrganizationRepositories( client: typeof graphql, org: string, + catalogPath: string, ): Promise<{ repositories: Repository[] }> { + const catalogPathRef = `HEAD:${catalogPath}`; const query = ` - query repositories($org: String!, $cursor: String) { + query repositories($org: String!, $catalogPathRef: String!, $cursor: String) { repositoryOwner(login: $org) { login repositories(first: 100, after: $cursor) { nodes { name + catalogInfoFile: object(expression: $catalogPathRef) { + __typename + ... on Blob { + id + } + } url isArchived repositoryTopics(first: 100) { @@ -302,7 +315,7 @@ export async function getOrganizationRepositories( query, r => r.repositoryOwner?.repositories, x => x, - { org }, + { org, catalogPathRef }, ); return { repositories }; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 63fb77288b..8b315a8a1e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -170,6 +170,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -267,6 +272,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -341,6 +351,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -381,6 +396,110 @@ describe('GitHubEntityProvider', () => { entities: expectedEntities, }); }); + + it('should filter out invalid locations when validateLocationsExist is set to true', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'catalog-custom.yaml', + filters: { + branch: 'main', + }, + validateLocationsExist: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: null, + }, + { + name: 'another-repo', + url: 'https://github.com/test-org/another-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/another-repo/blob/main/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-934f500db2ba2e8ea3524567926f45a73bb0b532', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); }); it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { @@ -437,6 +556,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-2', @@ -455,6 +579,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-3', @@ -470,6 +599,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3909b5a3c2..42dd6afa5d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -163,6 +163,7 @@ export class GitHubEntityProvider implements EntityProvider { private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; + const catalogPath = this.config.catalogPath; const orgUrl = `https://${host}/${organization}`; const { headers } = await this.githubCredentialsProvider.getCredentials({ @@ -177,8 +178,18 @@ export class GitHubEntityProvider implements EntityProvider { const { repositories } = await getOrganizationRepositories( client, organization, + catalogPath, ); + if (this.config.validateLocationsExist) { + return repositories.filter(repository => { + return ( + repository.catalogInfoFile?.__typename === 'Blob' && + repository.catalogInfoFile.text !== '' + ); + }); + } + return repositories; }