diff --git a/.changeset/big-jars-repair.md b/.changeset/big-jars-repair.md new file mode 100644 index 0000000000..63fef2c740 --- /dev/null +++ b/.changeset/big-jars-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +GitHub discovery processor adds support for discovering the default GitHub branch diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index ca8c702c8f..f8647cb5d2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -35,7 +35,8 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), @@ -43,14 +44,21 @@ describe('GithubDiscoveryProcessor', () => { org: 'foo', host: 'github.com', repoSearchPath: /^proj.*$/, - catalogPath: '/blob/master/catalog.yaml', + branch: 'master', + catalogPath: '/catalog.yaml', + }); + expect(parseUrl('https://github.com/foo')).toEqual({ + org: 'foo', + host: 'github.com', + repoSearchPath: /^.*$/, + branch: '-', + catalogPath: '/catalog-info.yaml', }); }); it('throws on incorrectly formed URLs', () => { expect(() => parseUrl('https://github.com')).toThrow(); expect(() => parseUrl('https://github.com//')).toThrow(); - expect(() => parseUrl('https://github.com/foo')).toThrow(); expect(() => parseUrl('https://github.com//foo')).toThrow(); expect(() => parseUrl('https://github.com/org/teams')).toThrow(); expect(() => parseUrl('https://github.com/org//teams')).toThrow(); @@ -125,11 +133,17 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'master', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -168,16 +182,25 @@ describe('GithubDiscoveryProcessor', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-cli', url: 'https://github.com/backstage/techdocs-cli', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'techdocs-container', url: 'https://github.com/backstage/techdocs-container', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); @@ -215,21 +238,33 @@ describe('GithubDiscoveryProcessor', () => { name: 'abstest', url: 'https://github.com/backstage/abctest', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test', url: 'https://github.com/backstage/test', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'test-archived', url: 'https://github.com/backstage/test', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, { name: 'testxyz', url: 'https://github.com/backstage/testxyz', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, ], }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index f7069a9f64..567f6536d0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -28,7 +28,18 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** * Extracts repositories out of a GitHub org. - */ + * + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. + * + * target: "https://github.com/backstage" + * or + * target: https://github.com/backstage/*\/blob/-/catalog-info.yaml + * + * You may also explicitly specify the source branch: + * + * target: https://github.com/backstage/*\/blob/main/catalog-info.yaml + **/ export class GithubDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; private readonly logger: Logger; @@ -65,7 +76,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); } - const { org, repoSearchPath, catalogPath, host } = parseUrl( + const { org, repoSearchPath, catalogPath, branch, host } = parseUrl( location.target, ); @@ -97,11 +108,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { ); for (const repository of matching) { + const path = `/blob/${ + branch === '-' ? repository.defaultBranchRef.name : branch + }${catalogPath}`; emit( results.location( { type: 'url', - target: `${repository.url}${catalogPath}`, + target: `${repository.url}${path}`, }, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting @@ -123,19 +137,31 @@ export function parseUrl(urlString: string): { org: string; repoSearchPath: RegExp; catalogPath: string; + branch: string; host: string; } { const url = new URL(urlString); const path = url.pathname.substr(1).split('/'); // /backstage/techdocs-*/blob/master/catalog-info.yaml + // can also be + // /backstage if (path.length > 2 && path[0].length && path[1].length) { return { org: decodeURIComponent(path[0]), repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), - catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, + branch: decodeURIComponent(path[3]), + catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`, host: url.host, }; + } else if (path.length === 1 && path[0].length) { + return { + org: decodeURIComponent(path[0]), + host: url.host, + repoSearchPath: escapeRegExp('*'), + catalogPath: '/catalog-info.yaml', + branch: '-', + }; } throw new Error(`Failed to parse ${urlString}`); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 11cdf671c3..e481355fb4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -201,11 +201,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], pageInfo: { @@ -221,11 +227,17 @@ describe('github', () => { name: 'backstage', url: 'https://github.com/backstage/backstage', isArchived: false, + defaultBranchRef: { + name: 'main', + }, }, { name: 'demo', url: 'https://github.com/backstage/demo', isArchived: true, + defaultBranchRef: { + name: 'main', + }, }, ], }; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 70479110c2..9eb5073710 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -60,6 +60,9 @@ export type Repository = { name: string; url: string; isArchived: boolean; + defaultBranchRef: { + name: string; + }; }; export type Connection = { @@ -252,6 +255,9 @@ export async function getOrganizationRepositories( name url isArchived + defaultBranchRef { + name + } } pageInfo { hasNextPage