diff --git a/.changeset/moody-mice-cheer.md b/.changeset/moody-mice-cheer.md new file mode 100644 index 0000000000..50d0e57b58 --- /dev/null +++ b/.changeset/moody-mice-cheer.md @@ -0,0 +1,15 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Added an option to scan GitHub for repositories using a new location type `github-discovery`. +Example: + +```yaml +type: 'github-discovery', +target: + 'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml' +``` + +You can use wildcards (`*`) as well. This will add `location` entities for each matching repository. +Currently though, you must specify the exact path of the `catalog.yaml` file in the repository. diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..41698e1d04 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -0,0 +1,222 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './github'; +import { ConfigReader } from '@backstage/config'; + +jest.mock('./github'); +const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< + typeof getOrganizationRepositories +>; + +describe('GithubDiscoveryProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /^proj$/, + catalogPath: '/blob/master/catalog.yaml', + }); + expect( + parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /^proj.*$/, + catalogPath: '/blob/master/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://github.com')).toThrow(); + expect(() => parseUrl('https://github.com//')).toThrow(); + expect(() => parseUrl('https://github.com/foo')).toThrow(); + expect(() => parseUrl('https://github.com//foo')).toThrow(); + expect(() => parseUrl('https://github.com/org/teams')).toThrow(); + expect(() => parseUrl('https://github.com/org//teams')).toThrow(); + }); + }); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-github-discovery', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'blob' }, + { host: 'ghe.example.net', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub integration that matches https:\/\/not.github.com\/apa/, + ); + }); + }); + + describe('handles repositories', () => { + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + + beforeEach(() => { + mockGetOrganizationRepositories.mockClear(); + }); + + it('output all repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { name: 'demo', url: 'https://github.com/backstage/demo' }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/demo/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + + it('output repositories with wildcards', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: + 'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'techdocs-cli', + url: 'https://github.com/backstage/techdocs-cli', + }, + { + name: 'techdocs-container', + url: 'https://github.com/backstage/techdocs-container', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + it('filter unrelated repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'abstest', url: 'https://github.com/backstage/abctest' }, + { + name: 'test', + url: 'https://github.com/backstage/test', + }, + { + name: 'testxyz', + url: 'https://github.com/backstage/testxyz', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts new file mode 100644 index 0000000000..9af247fd25 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -0,0 +1,130 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { getOrganizationRepositories } from './github'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * Extracts repositories out of a GitHub org. + */ +export class GithubDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new GithubDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'github-discovery') { + return false; + } + + const gitHubConfig = this.integrations.github.byUrl(location.target) + ?.config; + if (!gitHubConfig) { + throw new Error( + `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, + ); + } + const { headers } = await GithubCredentialsProvider.create( + gitHubConfig, + ).getCredentials({ url: location.target }); + const { org, repoSearchPath, catalogPath } = parseUrl(location.target); + + const client = graphql.defaults({ + baseUrl: gitHubConfig.apiBaseUrl, + headers, + }); + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info(`Reading GitHub repositories from ${location.target}`); + + const { repositories } = await getOrganizationRepositories(client, org); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${repositories.length} GitHub repositories in ${duration} seconds`, + ); + + for (const repository of repositories) { + if (!repoSearchPath.test(repository.name)) { + continue; + } + emit( + results.location( + { + type: 'url', + target: `${repository.url}${catalogPath}`, + }, + false, + ), + ); + } + + return true; + } +} + +/* + * Helpers + */ + +export function parseUrl( + urlString: string, +): { org: string; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + if (path.length > 2 && path[0].length && path[1].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), + catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 8911e84d99..81b44c706d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -22,6 +22,7 @@ import { getOrganizationTeams, getOrganizationUsers, getTeamMembers, + getOrganizationRepositories, QueryResponse, } from './github'; @@ -151,4 +152,48 @@ describe('github', () => { await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output); }); }); + + describe('getOrganizationRepositories', () => { + it('read repositories', async () => { + const input: QueryResponse = { + organization: { + repositories: { + nodes: [ + { + name: 'backstage', + url: 'https://github.com/backstage/backstage', + }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + pageInfo: { + hasNextPage: false, + }, + }, + }, + }; + + const output = { + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + }; + + server.use( + graphqlMsw.query('repositories', (_req, res, ctx) => + res(ctx.data(input)), + ), + ); + + await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( + output, + ); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 2b33d72f65..d50887c592 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -27,6 +27,7 @@ export type Organization = { membersWithRole?: Connection; team?: Team; teams?: Connection; + repositories?: Connection; }; export type PageInfo = { @@ -52,6 +53,11 @@ export type Team = { members: Connection; }; +export type Repository = { + name: string; + url: string; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -216,6 +222,39 @@ export async function getOrganizationTeams( return { groups, groupMemberUsers }; } +export async function getOrganizationRepositories( + client: typeof graphql, + org: string, +): Promise<{ repositories: Repository[] }> { + const query = ` + query repositories($org: String!, $cursor: String) { + organization(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + `; + + const repositories = await queryWithPaging( + client, + query, + r => r.organization?.repositories, + x => x, + { org }, + ); + + return { repositories }; +} + /** * Gets all the users out of a GitHub organization. * diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index e424d7bafb..2063e8c1b2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -16,4 +16,8 @@ export { readGithubConfig } from './config'; export type { ProviderConfig } from './config'; -export { getOrganizationTeams, getOrganizationUsers } from './github'; +export { + getOrganizationTeams, + getOrganizationUsers, + getOrganizationRepositories, +} from './github'; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 05dd327f0e..736c64ea39 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -21,6 +21,7 @@ export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAcco export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; +export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f7544ef42e..a75c7da6e8 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -43,6 +43,7 @@ import { CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, + GithubDiscoveryProcessor, GithubOrgReaderProcessor, HigherOrderOperation, HigherOrderOperations, @@ -303,6 +304,7 @@ export class CatalogBuilder { if (!this.processorsReplace) { processors.push( new FileReaderProcessor(), + GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),