From 84364b35c850a3fbee5a77383725b6107c69c4a0 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 28 Jan 2021 13:38:55 +0200 Subject: [PATCH] Added support for github-discovery locations --- .changeset/moody-mice-cheer.md | 5 + .../GithubDiscoveryProcessor.test.ts | 190 ++++++++++++++++++ .../processors/GithubDiscoveryProcessor.ts | 130 ++++++++++++ .../processors/github/github.test.ts | 45 +++++ .../src/ingestion/processors/github/github.ts | 46 ++++- .../src/ingestion/processors/github/index.ts | 6 +- 6 files changed, 420 insertions(+), 2 deletions(-) create mode 100644 .changeset/moody-mice-cheer.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts diff --git a/.changeset/moody-mice-cheer.md b/.changeset/moody-mice-cheer.md new file mode 100644 index 0000000000..bdbb0722b0 --- /dev/null +++ b/.changeset/moody-mice-cheer.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added an option to scan github for repositories using a new location type 'github-discovery' diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..a3713fb374 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -0,0 +1,190 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './github'; + +jest.mock('./github'); +const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< + typeof getOrganizationRepositories +>; + +describe('GithubOrgReaderProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /proj/, + catalogPath: 'blob/master/catalog.yaml', + }); + expect( + parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /proj.*/, + catalogPath: 'blob/master/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://github.com')).toThrow(); + expect(() => parseUrl('https://github.com//')).toThrow(); + expect(() => parseUrl('https://github.com/foo')).toThrow(); + expect(() => parseUrl('https://github.com//foo')).toThrow(); + expect(() => parseUrl('https://github.com/org/teams')).toThrow(); + expect(() => parseUrl('https://github.com/org//teams')).toThrow(); + }); + }); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + const location: LocationSpec = { + type: 'not-github-discovery', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, + ); + }); + }); + + describe('handles repositories', () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + + beforeEach(() => { + mockGetOrganizationRepositories.mockClear(); + }); + + it('output all repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { name: 'demo', url: 'https://github.com/backstage/demo' }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/demo/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + + it('filter unrelated repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: + 'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'techdocs-cli', + url: 'https://github.com/backstage/techdocs-cli', + }, + { + name: 'techdocs-container', + url: 'https://github.com/backstage/techdocs-container', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts new file mode 100644 index 0000000000..e5590b6179 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -0,0 +1,130 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { + getOrganizationRepositories, + ProviderConfig, + readGithubConfig, +} from './github'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * Extracts teams and users out of a GitHub org. + */ +export class GithubDiscoveryProcessor implements CatalogProcessor { + private readonly providers: ProviderConfig[]; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + return new GithubDiscoveryProcessor({ + ...options, + providers: readGithubConfig(config), + }); + } + + constructor(options: { providers: ProviderConfig[]; logger: Logger }) { + this.providers = options.providers; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'github-discovery') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + ); + } + + const { org, repoSearchPath, catalogPath } = parseUrl(location.target); + const client = !provider.token + ? graphql + : graphql.defaults({ + baseUrl: provider.apiBaseUrl, + headers: { + authorization: `token ${provider.token}`, + }, + }); + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading GitHub repositories'); + + const { repositories } = await getOrganizationRepositories(client, org); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${repositories.length} GitHub repositories in ${duration} seconds`, + ); + + for (const repository of repositories) { + if (!repoSearchPath.test(repository.name)) { + continue; + } + emit( + results.location( + { + type: 'url', + target: `${repository.url}/${catalogPath}`, + }, + false, + ), + ); + } + + return true; + } +} + +/* + * Helpers + */ + +export function parseUrl( + urlString: string, +): { org: string; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + if (path.length > 2 && path[0].length && path[1].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), + catalogPath: decodeURIComponent(path.slice(2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(str.replace(/\*/g, '.*')); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 8911e84d99..81b44c706d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -22,6 +22,7 @@ import { getOrganizationTeams, getOrganizationUsers, getTeamMembers, + getOrganizationRepositories, QueryResponse, } from './github'; @@ -151,4 +152,48 @@ describe('github', () => { await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output); }); }); + + describe('getOrganizationRepositories', () => { + it('read repositories', async () => { + const input: QueryResponse = { + organization: { + repositories: { + nodes: [ + { + name: 'backstage', + url: 'https://github.com/backstage/backstage', + }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + pageInfo: { + hasNextPage: false, + }, + }, + }, + }; + + const output = { + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + }; + + server.use( + graphqlMsw.query('repositories', (_req, res, ctx) => + res(ctx.data(input)), + ), + ); + + await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( + output, + ); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 2b33d72f65..cd4719c46f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -14,8 +14,13 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GroupEntity, + LocationEntity, + UserEntity, +} from '@backstage/catalog-model'; import { graphql } from '@octokit/graphql'; +import { xorWith } from 'lodash/fp'; // Graphql types @@ -27,6 +32,7 @@ export type Organization = { membersWithRole?: Connection; team?: Team; teams?: Connection; + repositories?: Connection; }; export type PageInfo = { @@ -52,6 +58,11 @@ export type Team = { members: Connection; }; +export type Repository = { + name: string; + url: string; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -216,6 +227,39 @@ export async function getOrganizationTeams( return { groups, groupMemberUsers }; } +export async function getOrganizationRepositories( + client: typeof graphql, + org: string, +): Promise<{ repositories: Repository[] }> { + const query = ` + query repositories($org: String!, $cursor: String) { + organization(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + `; + + const repositories = await queryWithPaging( + client, + query, + r => r.organization?.repositories, + x => x, + { org }, + ); + + return { repositories }; +} + /** * Gets all the users out of a GitHub organization. * diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index e424d7bafb..2063e8c1b2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -16,4 +16,8 @@ export { readGithubConfig } from './config'; export type { ProviderConfig } from './config'; -export { getOrganizationTeams, getOrganizationUsers } from './github'; +export { + getOrganizationTeams, + getOrganizationUsers, + getOrganizationRepositories, +} from './github';