From c531345df68d943c4652c30b898f9c6e7d40de7e Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 18:44:21 +0100 Subject: [PATCH 1/5] ability to discover catalog-infos from default branch Previously the github-discovery supported only repos from a specific branch. This change cause the discovery to work in two modes. It continues to work as it did with a wildcard for repositories on a specified branch. The new mode added causes the discovery processor to use the default branch and looks for a file called catalog-info.yaml. Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 5 ++- .../processors/GithubDiscoveryProcessor.ts | 32 +++++++++++++++---- .../src/ingestion/processors/github/github.ts | 6 ++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index ca8c702c8f..0717e24452 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -45,12 +45,15 @@ describe('GithubDiscoveryProcessor', () => { repoSearchPath: /^proj.*$/, catalogPath: '/blob/master/catalog.yaml', }); + expect(parseUrl('https://github.com/foo')).toEqual({ + org: 'foo', + host: 'github.com', + }); }); it('throws on incorrectly formed URLs', () => { expect(() => parseUrl('https://github.com')).toThrow(); expect(() => parseUrl('https://github.com//')).toThrow(); - expect(() => parseUrl('https://github.com/foo')).toThrow(); expect(() => parseUrl('https://github.com//foo')).toThrow(); expect(() => parseUrl('https://github.com/org/teams')).toThrow(); expect(() => parseUrl('https://github.com/org//teams')).toThrow(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index f7069a9f64..8333728fb9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -28,7 +28,15 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** * Extracts repositories out of a GitHub org. - */ + * + * It can be configured in two modes. The first will create locations for all + * catalog-info.yaml files on the default branch. The second will create locations + * for all projects which have a catalog-info.yaml on the master branch. + * + * target: "https://github.com/backstage" + * or + * target: https://github.com/backstage/*\/blob/master/catalog-info.yaml + **/ export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; @@ -87,9 +95,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); - const matching = repositories.filter( - r => !r.isArchived && repoSearchPath.test(r.name), - ); + const matching = repoSearchPath + ? repositories.filter(r => !r.isArchived && repoSearchPath.test(r.name)) + : repositories; const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( @@ -97,11 +105,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); for (const repository of matching) { + const path = catalogPath + ? catalogPath + : `/blob/${repository.defaultBranchRef.name}/catalog-info.yaml`; emit( results.location( { type: 'url', - target: `${repository.url}${catalogPath}`, + target: `${repository.url}${path}`, }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting @@ -121,14 +132,16 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { export function parseUrl(urlString: string): { org: string; - repoSearchPath: RegExp; - catalogPath: string; + repoSearchPath?: RegExp; + catalogPath?: string; host: string; } { const url = new URL(urlString); const path = url.pathname.substr(1).split('/'); // /backstage/techdocs-*/blob/master/catalog-info.yaml + // can also be + // /backstage if (path.length > 2 && path[0].length && path[1].length) { return { org: decodeURIComponent(path[0]), @@ -136,6 +149,11 @@ export function parseUrl(urlString: string): { catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, host: url.host, }; + } else if (path.length === 1 && path[0].length) { + return { + org: decodeURIComponent(path[0]), + host: url.host, + }; } throw new Error(`Failed to parse ${urlString}`); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 70479110c2..9eb5073710 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -60,6 +60,9 @@ export type Repository = { name: string; url: string; isArchived: boolean; + defaultBranchRef: { + name: string; + }; }; export type Connection = { @@ -252,6 +255,9 @@ export async function getOrganizationRepositories( name url isArchived + defaultBranchRef { + name + } } pageInfo { hasNextPage From 8b39242c4850725fe1b4661019c0ece7d0c05382 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 20:52:05 +0100 Subject: [PATCH 2/5] adds changeset for github discovery change Signed-off-by: Brian Fletcher --- .changeset/big-jars-repair.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/big-jars-repair.md diff --git a/.changeset/big-jars-repair.md b/.changeset/big-jars-repair.md new file mode 100644 index 0000000000..63fef2c740 --- /dev/null +++ b/.changeset/big-jars-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +GitHub discovery processor adds support for discovering the default GitHub branch From f337c2fa2fe75df9696b0c2e58d78f5fb18f9113 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 20:54:41 +0100 Subject: [PATCH 3/5] fixes tests types Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 0717e24452..7ebb5b0da1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -128,11 +128,17 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'master', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -171,16 +177,25 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -218,21 +233,33 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test-archived', url: 'https://github.com/backstage/test', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, { name: 'testxyz', url: 'https://github.com/backstage/testxyz', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); From 1a98eb41e6e096d182f185a72cdcf18246ad2e17 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 23 Aug 2021 21:08:01 +0100 Subject: [PATCH 4/5] fixing more tests mocks Signed-off-by: Brian Fletcher --- .../src/ingestion/processors/github/github.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 11cdf671c3..e481355fb4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -201,11 +201,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], pageInfo: { @@ -221,11 +227,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], }; From b000c79abfefe506f3173565fde7e7ad681382da Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 24 Aug 2021 16:23:55 +0100 Subject: [PATCH 5/5] Support full gamit of path options for discovery It'll now support anything from: https://github.com/backstage to https://github.com/backstage/*\/blob/-/catalog-info.yaml Signed-off-by: Brian Fletcher --- .../GithubDiscoveryProcessor.test.ts | 9 +++-- .../processors/GithubDiscoveryProcessor.ts | 36 +++++++++++-------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 7ebb5b0da1..f8647cb5d2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -35,7 +35,8 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), @@ -43,11 +44,15 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj.*$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', }); expect(parseUrl('https://github.com/foo')).toEqual({ org: 'foo', host: 'github.com', + repoSearchPath: /^.*$/, + branch: '-', + catalogPath: '/catalog-info.yaml', }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 8333728fb9..567f6536d0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -29,13 +29,16 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** * Extracts repositories out of a GitHub org. * - * It can be configured in two modes. The first will create locations for all - * catalog-info.yaml files on the default branch. The second will create locations - * for all projects which have a catalog-info.yaml on the master branch. + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. * * target: "https://github.com/backstage" * or - * target: https://github.com/backstage/*\/blob/master/catalog-info.yaml + * target: https://github.com/backstage/*\/blob/-/catalog-info.yaml + * + * You may also explicitly specify the source branch: + * + * target: https://github.com/backstage/*\/blob/main/catalog-info.yaml **/ export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -73,7 +76,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); } - const { org, repoSearchPath, catalogPath, host } = parseUrl( + const { org, repoSearchPath, catalogPath, branch, host } = parseUrl( location.target, ); @@ -95,9 +98,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); - const matching = repoSearchPath - ? repositories.filter(r => !r.isArchived && repoSearchPath.test(r.name)) - : repositories; + const matching = repositories.filter( + r => !r.isArchived && repoSearchPath.test(r.name), + ); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( @@ -105,9 +108,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); for (const repository of matching) { - const path = catalogPath - ? catalogPath - : `/blob/${repository.defaultBranchRef.name}/catalog-info.yaml`; + const path = `/blob/${ + branch === '-' ? repository.defaultBranchRef.name : branch + }${catalogPath}`; emit( results.location( { @@ -132,8 +135,9 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { export function parseUrl(urlString: string): { org: string; - repoSearchPath?: RegExp; - catalogPath?: string; + repoSearchPath: RegExp; + catalogPath: string; + branch: string; host: string; } { const url = new URL(urlString); @@ -146,13 +150,17 @@ export function parseUrl(urlString: string): { return { org: decodeURIComponent(path[0]), repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), - catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, + branch: decodeURIComponent(path[3]), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, host: url.host, }; } else if (path.length === 1 && path[0].length) { return { org: decodeURIComponent(path[0]), host: url.host, + repoSearchPath: escapeRegExp('*'), + catalogPath: '/catalog-info.yaml', + branch: '-', }; }