From 84364b35c850a3fbee5a77383725b6107c69c4a0 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 28 Jan 2021 13:38:55 +0200 Subject: [PATCH 1/8] Added support for github-discovery locations --- .changeset/moody-mice-cheer.md | 5 + .../GithubDiscoveryProcessor.test.ts | 190 ++++++++++++++++++ .../processors/GithubDiscoveryProcessor.ts | 130 ++++++++++++ .../processors/github/github.test.ts | 45 +++++ .../src/ingestion/processors/github/github.ts | 46 ++++- .../src/ingestion/processors/github/index.ts | 6 +- 6 files changed, 420 insertions(+), 2 deletions(-) create mode 100644 .changeset/moody-mice-cheer.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts diff --git a/.changeset/moody-mice-cheer.md b/.changeset/moody-mice-cheer.md new file mode 100644 index 0000000000..bdbb0722b0 --- /dev/null +++ b/.changeset/moody-mice-cheer.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added an option to scan github for repositories using a new location type 'github-discovery' diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..a3713fb374 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -0,0 +1,190 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './github'; + +jest.mock('./github'); +const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< + typeof getOrganizationRepositories +>; + +describe('GithubOrgReaderProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /proj/, + catalogPath: 'blob/master/catalog.yaml', + }); + expect( + parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), + ).toEqual({ + org: 'foo', + repoSearchPath: /proj.*/, + catalogPath: 'blob/master/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://github.com')).toThrow(); + expect(() => parseUrl('https://github.com//')).toThrow(); + expect(() => parseUrl('https://github.com/foo')).toThrow(); + expect(() => parseUrl('https://github.com//foo')).toThrow(); + expect(() => parseUrl('https://github.com/org/teams')).toThrow(); + expect(() => parseUrl('https://github.com/org//teams')).toThrow(); + }); + }); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + const location: LocationSpec = { + type: 'not-github-discovery', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, + ); + }); + }); + + describe('handles repositories', () => { + const processor = new GithubDiscoveryProcessor({ + providers: [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + logger: getVoidLogger(), + }); + + beforeEach(() => { + mockGetOrganizationRepositories.mockClear(); + }); + + it('output all repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { name: 'demo', url: 'https://github.com/backstage/demo' }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/demo/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + + it('filter unrelated repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: + 'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'techdocs-cli', + url: 'https://github.com/backstage/techdocs-cli', + }, + { + name: 'techdocs-container', + url: 'https://github.com/backstage/techdocs-container', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-cli/blob/master/catalog.yaml', + }, + optional: false, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://github.com/backstage/techdocs-container/blob/master/catalog.yaml', + }, + optional: false, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts new file mode 100644 index 0000000000..e5590b6179 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -0,0 +1,130 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; +import { + getOrganizationRepositories, + ProviderConfig, + readGithubConfig, +} from './github'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * Extracts teams and users out of a GitHub org. + */ +export class GithubDiscoveryProcessor implements CatalogProcessor { + private readonly providers: ProviderConfig[]; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + return new GithubDiscoveryProcessor({ + ...options, + providers: readGithubConfig(config), + }); + } + + constructor(options: { providers: ProviderConfig[]; logger: Logger }) { + this.providers = options.providers; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'github-discovery') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + ); + } + + const { org, repoSearchPath, catalogPath } = parseUrl(location.target); + const client = !provider.token + ? graphql + : graphql.defaults({ + baseUrl: provider.apiBaseUrl, + headers: { + authorization: `token ${provider.token}`, + }, + }); + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading GitHub repositories'); + + const { repositories } = await getOrganizationRepositories(client, org); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${repositories.length} GitHub repositories in ${duration} seconds`, + ); + + for (const repository of repositories) { + if (!repoSearchPath.test(repository.name)) { + continue; + } + emit( + results.location( + { + type: 'url', + target: `${repository.url}/${catalogPath}`, + }, + false, + ), + ); + } + + return true; + } +} + +/* + * Helpers + */ + +export function parseUrl( + urlString: string, +): { org: string; repoSearchPath: RegExp; catalogPath: string } { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // /backstage/techdocs-*/blob/master/catalog-info.yaml + if (path.length > 2 && path[0].length && path[1].length) { + return { + org: decodeURIComponent(path[0]), + repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), + catalogPath: decodeURIComponent(path.slice(2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(str.replace(/\*/g, '.*')); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 8911e84d99..81b44c706d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -22,6 +22,7 @@ import { getOrganizationTeams, getOrganizationUsers, getTeamMembers, + getOrganizationRepositories, QueryResponse, } from './github'; @@ -151,4 +152,48 @@ describe('github', () => { await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output); }); }); + + describe('getOrganizationRepositories', () => { + it('read repositories', async () => { + const input: QueryResponse = { + organization: { + repositories: { + nodes: [ + { + name: 'backstage', + url: 'https://github.com/backstage/backstage', + }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + pageInfo: { + hasNextPage: false, + }, + }, + }, + }; + + const output = { + repositories: [ + { name: 'backstage', url: 'https://github.com/backstage/backstage' }, + { + name: 'demo', + url: 'https://github.com/backstage/demo', + }, + ], + }; + + server.use( + graphqlMsw.query('repositories', (_req, res, ctx) => + res(ctx.data(input)), + ), + ); + + await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( + output, + ); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 2b33d72f65..cd4719c46f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -14,8 +14,13 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GroupEntity, + LocationEntity, + UserEntity, +} from '@backstage/catalog-model'; import { graphql } from '@octokit/graphql'; +import { xorWith } from 'lodash/fp'; // Graphql types @@ -27,6 +32,7 @@ export type Organization = { membersWithRole?: Connection; team?: Team; teams?: Connection; + repositories?: Connection; }; export type PageInfo = { @@ -52,6 +58,11 @@ export type Team = { members: Connection; }; +export type Repository = { + name: string; + url: string; +}; + export type Connection = { pageInfo: PageInfo; nodes: T[]; @@ -216,6 +227,39 @@ export async function getOrganizationTeams( return { groups, groupMemberUsers }; } +export async function getOrganizationRepositories( + client: typeof graphql, + org: string, +): Promise<{ repositories: Repository[] }> { + const query = ` + query repositories($org: String!, $cursor: String) { + organization(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + `; + + const repositories = await queryWithPaging( + client, + query, + r => r.organization?.repositories, + x => x, + { org }, + ); + + return { repositories }; +} + /** * Gets all the users out of a GitHub organization. * diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index e424d7bafb..2063e8c1b2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -16,4 +16,8 @@ export { readGithubConfig } from './config'; export type { ProviderConfig } from './config'; -export { getOrganizationTeams, getOrganizationUsers } from './github'; +export { + getOrganizationTeams, + getOrganizationUsers, + getOrganizationRepositories, +} from './github'; From af045e3a77ac63775c9ed70cc9b2d77f2c26d675 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 28 Jan 2021 14:37:44 +0200 Subject: [PATCH 2/8] Extended changelist docs; Fixed lint errors --- .changeset/moody-mice-cheer.md | 12 +++++++++++- .../src/ingestion/processors/github/github.ts | 7 +------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.changeset/moody-mice-cheer.md b/.changeset/moody-mice-cheer.md index bdbb0722b0..f2e5d29ba1 100644 --- a/.changeset/moody-mice-cheer.md +++ b/.changeset/moody-mice-cheer.md @@ -2,4 +2,14 @@ '@backstage/plugin-catalog-backend': minor --- -Added an option to scan github for repositories using a new location type 'github-discovery' +Added an option to scan GitHub for repositories using a new location type `github-discovery`. +Example: + +```yaml +type: 'github-discovery', +target: + 'https://github.com/backstage/techdocs-*/blob/master/catalog.yaml' +``` + +You can use wildcards (`*`) as well. This will add `location` entities for each matching repository. +Currently though, you must specify the exact path of the `catalog.yaml` file in the repository. diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index cd4719c46f..d50887c592 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -14,13 +14,8 @@ * limitations under the License. */ -import { - GroupEntity, - LocationEntity, - UserEntity, -} from '@backstage/catalog-model'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { graphql } from '@octokit/graphql'; -import { xorWith } from 'lodash/fp'; // Graphql types From 5f305b914132d4cb8aff1c6a99fd569f4242b72c Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 28 Jan 2021 15:00:16 +0200 Subject: [PATCH 3/8] Registered GithubDiscoveryProcessor in CatalogBuilder --- plugins/catalog-backend/src/ingestion/processors/index.ts | 1 + plugins/catalog-backend/src/service/CatalogBuilder.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index a7b00d7065..77dfb67687 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -21,6 +21,7 @@ export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAcco export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; +export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationRefProcessor } from './LocationEntityProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f6170135fd..22865be66d 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -41,6 +41,7 @@ import { CatalogProcessor, CodeOwnersProcessor, FileReaderProcessor, + GithubDiscoveryProcessor, GithubOrgReaderProcessor, HigherOrderOperation, HigherOrderOperations, @@ -281,6 +282,7 @@ export class CatalogBuilder { if (!this.processorsReplace) { processors.push( new FileReaderProcessor(), + GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), From 7413148c5f53683d5321227a7fb27a5fa8739b68 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Fri, 29 Jan 2021 19:04:34 +0200 Subject: [PATCH 4/8] Changed to use Github integration configs --- plugins/catalog-backend/package.json | 1 + .../GithubDiscoveryProcessor.test.ts | 54 ++++++++++++------- .../processors/GithubDiscoveryProcessor.ts | 54 +++++++++++-------- yarn.lock | 49 +++++++++++++---- 4 files changed, 106 insertions(+), 52 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index b2db27770b..07fd000618 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -34,6 +34,7 @@ "@backstage/backend-common": "^0.5.0", "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", + "@backstage/integration": "^0.3.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index a3713fb374..9daecb26d3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -56,12 +56,15 @@ describe('GithubOrgReaderProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + ]), logger: getVoidLogger(), }); const location: LocationSpec = { @@ -75,12 +78,22 @@ describe('GithubOrgReaderProcessor', () => { it('rejects unknown targets', async () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + [ + 'ghe.example.net', + { + host: 'ghe.example.net', + apiBaseUrl: 'https://ghe.example.net/api/v3', + }, + ], + ]), logger: getVoidLogger(), }); const location: LocationSpec = { @@ -90,19 +103,22 @@ describe('GithubOrgReaderProcessor', () => { await expect( processor.readLocation(location, false, () => {}), ).rejects.toThrow( - /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, + /There is no GitHub integration that matches https:\/\/not.github.com\/apa/, ); }); }); describe('handles repositories', () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + ]), logger: getVoidLogger(), }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index e5590b6179..698f0ad987 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -16,32 +16,41 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { - getOrganizationRepositories, - ProviderConfig, - readGithubConfig, -} from './github'; +import { getOrganizationRepositories } from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** - * Extracts teams and users out of a GitHub org. + * Extracts repositories out of a GitHub org. */ export class GithubDiscoveryProcessor implements CatalogProcessor { - private readonly providers: ProviderConfig[]; + private readonly gitHubConfigMap: Map; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { + const configs = readGitHubIntegrationConfigs( + config.getOptionalConfigArray('integrations.github') ?? [], + ); + const gitHubConfigMap = new Map(configs.map(c => [c.host, c])); + return new GithubDiscoveryProcessor({ ...options, - providers: readGithubConfig(config), + gitHubConfigMap, }); } - constructor(options: { providers: ProviderConfig[]; logger: Logger }) { - this.providers = options.providers; + constructor(options: { + gitHubConfigMap: Map; + logger: Logger; + }) { + this.gitHubConfigMap = options.gitHubConfigMap; this.logger = options.logger; } @@ -54,24 +63,23 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { return false; } - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), + const gitHubConfig = this.gitHubConfigMap.get( + new URL(location.target).hostname, ); - if (!provider) { + if (!gitHubConfig) { throw new Error( - `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, ); } - + const { headers } = await GithubCredentialsProvider.create( + gitHubConfig, + ).getCredentials({ url: location.target }); const { org, repoSearchPath, catalogPath } = parseUrl(location.target); - const client = !provider.token - ? graphql - : graphql.defaults({ - baseUrl: provider.apiBaseUrl, - headers: { - authorization: `token ${provider.token}`, - }, - }); + + const client = graphql.defaults({ + baseUrl: gitHubConfig.apiBaseUrl, + headers, + }); // Read out all of the raw data const startTimestamp = Date.now(); diff --git a/yarn.lock b/yarn.lock index 7c743bc5b3..810090a45f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2473,9 +2473,11 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.7.0" + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" + integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== dependencies: - "@backstage/config" "^0.1.2" + "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2484,9 +2486,11 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.7.0" + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" + integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== dependencies: - "@backstage/config" "^0.1.2" + "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2495,16 +2499,17 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.5.0" + version "0.3.2" + resolved "https://registry.npmjs.org/@backstage/core/-/core-0.3.2.tgz#a8209126d5076cf4a8b9bd632fe4e5e2edb62916" + integrity sha512-i5d+Wh8js4qEWoAsPY5L7HVSWpumr1OhfF2dUCGYdyW6AMqVJPca6+n6zp1Rg2CO+J9norp44XAVVCbyhtUpig== dependencies: - "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.8" - "@backstage/theme" "^0.2.2" + "@backstage/config" "^0.1.1" + "@backstage/core-api" "^0.2.1" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" "@types/dagre" "^0.7.44" - "@types/prop-types" "^15.7.3" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" @@ -2513,7 +2518,7 @@ d3-shape "^2.0.0" d3-zoom "^2.0.0" dagre "^0.8.5" - immer "^8.0.1" + immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" @@ -2532,6 +2537,18 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" +"@backstage/integration@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/integration/-/integration-0.3.1.tgz#4bdf1883b7149f7d4866bf1732d8e371f37ed143" + integrity sha512-aHfZ83z3XmO3R1SPG3r3W9KjSab/aFJ+51LjvRr6OCthfFTIusEmFq4cro5RUn7IPTn/0L8qpBSW2xH6FlRuVw== + dependencies: + "@backstage/config" "^0.1.2" + "@octokit/auth-app" "^2.10.5" + "@octokit/rest" "^18.0.12" + cross-fetch "^3.0.6" + git-url-parse "^11.4.4" + luxon "^1.25.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -14180,6 +14197,13 @@ git-url-parse@^11.4.3: dependencies: git-up "^4.0.0" +git-url-parse@^11.4.4: + version "11.4.4" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.4.tgz#5d747debc2469c17bc385719f7d0427802d83d77" + integrity sha512-Y4o9o7vQngQDIU9IjyCmRJBin5iYjI5u9ZITnddRZpD7dcCFQj2sL2XuMNbLRE4b4B/4ENPsp2Q8P44fjAZ0Pw== + dependencies: + git-up "^4.0.0" + gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" @@ -15309,6 +15333,11 @@ immer@1.10.0: resolved "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" integrity sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg== +immer@^7.0.9: + version "7.0.15" + resolved "https://registry.npmjs.org/immer/-/immer-7.0.15.tgz#dc3bc6db87401659d2e737c67a21b227c484a4ad" + integrity sha512-yM7jo9+hvYgvdCQdqvhCNRRio0SCXc8xDPzA25SvKWa7b1WVPjLwQs1VYU5JPXjcJPTqAa5NP5dqpORGYBQ2AA== + immer@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656" From df5e7d4e43f2562fb72f463376025c89116f2abc Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 4 Feb 2021 17:40:35 +0200 Subject: [PATCH 5/8] Fixes following CR --- .../GithubDiscoveryProcessor.test.ts | 110 ++++++++++-------- .../processors/GithubDiscoveryProcessor.ts | 28 ++--- yarn.lock | 30 ++--- 3 files changed, 83 insertions(+), 85 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 9daecb26d3..fe6bb51521 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -18,27 +18,28 @@ import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from './github'; +import { ConfigReader } from '@backstage/config'; jest.mock('./github'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories >; -describe('GithubOrgReaderProcessor', () => { +describe('GithubDiscoveryProcessor', () => { describe('parseUrl', () => { it('parses well formed URLs', () => { expect( parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'), ).toEqual({ org: 'foo', - repoSearchPath: /proj/, + repoSearchPath: /^proj$/, catalogPath: 'blob/master/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), ).toEqual({ org: 'foo', - repoSearchPath: /proj.*/, + repoSearchPath: /^proj.*$/, catalogPath: 'blob/master/catalog.yaml', }); }); @@ -55,18 +56,14 @@ describe('GithubOrgReaderProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); const location: LocationSpec = { type: 'not-github-discovery', target: 'https://github.com', @@ -77,25 +74,17 @@ describe('GithubOrgReaderProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - [ - 'ghe.example.net', - { - host: 'ghe.example.net', - apiBaseUrl: 'https://ghe.example.net/api/v3', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'blob' }, + { host: 'ghe.example.net', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); const location: LocationSpec = { type: 'github-discovery', target: 'https://not.github.com/apa', @@ -109,18 +98,14 @@ describe('GithubOrgReaderProcessor', () => { }); describe('handles repositories', () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); beforeEach(() => { mockGetOrganizationRepositories.mockClear(); @@ -160,7 +145,7 @@ describe('GithubOrgReaderProcessor', () => { }); }); - it('filter unrelated repositories', async () => { + it('output repositories with wildcards', async () => { const location: LocationSpec = { type: 'github-discovery', target: @@ -202,5 +187,36 @@ describe('GithubOrgReaderProcessor', () => { optional: false, }); }); + it('filter unrelated repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'abstest', url: 'https://github.com/backstage/abctest' }, + { + name: 'test', + url: 'https://github.com/backstage/test', + }, + { + name: 'testxyz', + url: 'https://github.com/backstage/testxyz', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }, + optional: false, + }); + }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 698f0ad987..f96943eebb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -18,8 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GithubCredentialsProvider, - GitHubIntegrationConfig, - readGitHubIntegrationConfigs, + ScmIntegrations, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -31,26 +30,20 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; * Extracts repositories out of a GitHub org. */ export class GithubDiscoveryProcessor implements CatalogProcessor { - private readonly gitHubConfigMap: Map; + private readonly integrations: ScmIntegrations; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { - const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [], - ); - const gitHubConfigMap = new Map(configs.map(c => [c.host, c])); + const integrations = ScmIntegrations.fromConfig(config); return new GithubDiscoveryProcessor({ ...options, - gitHubConfigMap, + integrations, }); } - constructor(options: { - gitHubConfigMap: Map; - logger: Logger; - }) { - this.gitHubConfigMap = options.gitHubConfigMap; + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; this.logger = options.logger; } @@ -63,9 +56,8 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { return false; } - const gitHubConfig = this.gitHubConfigMap.get( - new URL(location.target).hostname, - ); + const gitHubConfig = this.integrations.github.byUrl(location.target) + ?.config; if (!gitHubConfig) { throw new Error( `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, @@ -83,7 +75,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { // Read out all of the raw data const startTimestamp = Date.now(); - this.logger.info('Reading GitHub repositories'); + this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); @@ -134,5 +126,5 @@ export function parseUrl( } export function escapeRegExp(str: string): RegExp { - return new RegExp(str.replace(/\*/g, '.*')); + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } diff --git a/yarn.lock b/yarn.lock index 810090a45f..c20a8c505d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2473,11 +2473,9 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" - integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== + version "0.7.0" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2486,11 +2484,9 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" - integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== + version "0.7.0" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2499,17 +2495,16 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.3.2" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.3.2.tgz#a8209126d5076cf4a8b9bd632fe4e5e2edb62916" - integrity sha512-i5d+Wh8js4qEWoAsPY5L7HVSWpumr1OhfF2dUCGYdyW6AMqVJPca6+n6zp1Rg2CO+J9norp44XAVVCbyhtUpig== + version "0.5.0" dependencies: - "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" + "@backstage/config" "^0.1.2" + "@backstage/core-api" "^0.2.8" + "@backstage/theme" "^0.2.2" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" "@types/dagre" "^0.7.44" + "@types/prop-types" "^15.7.3" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" @@ -2518,7 +2513,7 @@ d3-shape "^2.0.0" d3-zoom "^2.0.0" dagre "^0.8.5" - immer "^7.0.9" + immer "^8.0.1" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" @@ -15333,11 +15328,6 @@ immer@1.10.0: resolved "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" integrity sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg== -immer@^7.0.9: - version "7.0.15" - resolved "https://registry.npmjs.org/immer/-/immer-7.0.15.tgz#dc3bc6db87401659d2e737c67a21b227c484a4ad" - integrity sha512-yM7jo9+hvYgvdCQdqvhCNRRio0SCXc8xDPzA25SvKWa7b1WVPjLwQs1VYU5JPXjcJPTqAa5NP5dqpORGYBQ2AA== - immer@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656" From d742b1fcd5722c0fad4abf8b72d9fb36cdb6d654 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 4 Feb 2021 17:44:57 +0200 Subject: [PATCH 6/8] Rebased and reinstalled yarn.lock --- yarn.lock | 83 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index 713ef89c0a..90e6f63ccd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2583,7 +2583,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.7.0" + version "0.7.1" dependencies: "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" @@ -2595,7 +2595,7 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.7.0" + version "0.7.1" dependencies: "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" @@ -2607,11 +2607,11 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.5.0" + version "0.6.0" dependencies: "@backstage/config" "^0.1.2" "@backstage/core-api" "^0.2.8" - "@backstage/theme" "^0.2.2" + "@backstage/theme" "^0.2.3" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" @@ -2645,17 +2645,69 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/integration@^0.3.1": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/integration/-/integration-0.3.1.tgz#4bdf1883b7149f7d4866bf1732d8e371f37ed143" - integrity sha512-aHfZ83z3XmO3R1SPG3r3W9KjSab/aFJ+51LjvRr6OCthfFTIusEmFq4cro5RUn7IPTn/0L8qpBSW2xH6FlRuVw== +"@backstage/core@^0.5.0": + version "0.6.0" dependencies: "@backstage/config" "^0.1.2" - "@octokit/auth-app" "^2.10.5" - "@octokit/rest" "^18.0.12" - cross-fetch "^3.0.6" + "@backstage/core-api" "^0.2.8" + "@backstage/theme" "^0.2.3" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@testing-library/react-hooks" "^3.4.2" + "@types/dagre" "^0.7.44" + "@types/prop-types" "^15.7.3" + "@types/react" "^16.9" + "@types/react-sparklines" "^1.7.0" + classnames "^2.2.6" + clsx "^1.1.0" + d3-selection "^2.0.0" + d3-shape "^2.0.0" + d3-zoom "^2.0.0" + dagre "^0.8.5" + immer "^8.0.1" + lodash "^4.17.15" + material-table "^1.69.1" + prop-types "^15.7.2" + qs "^6.9.4" + rc-progress "^3.0.0" + react "^16.12.0" + react-dom "^16.12.0" + react-helmet "6.1.0" + react-hook-form "^6.6.0" + react-markdown "^5.0.2" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-sparklines "^1.7.0" + react-syntax-highlighter "^13.5.1" + react-use "^15.3.3" + remark-gfm "^1.0.0" + zen-observable "^0.8.15" + +"@backstage/plugin-catalog@^0.2.0", "@backstage/plugin-catalog@^0.2.1": + version "0.2.14" + resolved "https://registry.npmjs.org/@backstage/plugin-catalog/-/plugin-catalog-0.2.14.tgz#50a4176a55ffa543a426ec78cbc9deaecdbcf2b7" + integrity sha512-lDmNcC+m1zbbzYATUp5yIZ5PUp+YyBc1KKu3CCgqjLWSbJ1aJrU1N4g59euel1l2+qSW+lH76Kkp6ZYpZbSO9A== + dependencies: + "@backstage/catalog-client" "^0.3.5" + "@backstage/catalog-model" "^0.7.0" + "@backstage/core" "^0.5.0" + "@backstage/plugin-scaffolder" "^0.4.1" + "@backstage/theme" "^0.2.2" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@types/react" "^16.9" + classnames "^2.2.6" git-url-parse "^11.4.4" - luxon "^1.25.0" + moment "^2.26.0" + react "^16.13.1" + react-dom "^16.13.1" + react-helmet "6.1.0" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-use "^15.3.3" + swr "^0.3.0" "@bcoe/v8-coverage@^0.2.3": version "0.2.3" @@ -14360,13 +14412,6 @@ git-url-parse@^11.4.4: dependencies: git-up "^4.0.0" -git-url-parse@^11.4.4: - version "11.4.4" - resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.4.tgz#5d747debc2469c17bc385719f7d0427802d83d77" - integrity sha512-Y4o9o7vQngQDIU9IjyCmRJBin5iYjI5u9ZITnddRZpD7dcCFQj2sL2XuMNbLRE4b4B/4ENPsp2Q8P44fjAZ0Pw== - dependencies: - git-up "^4.0.0" - gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" From 61746174054d50936b69500414e195edcaf12841 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Tue, 9 Feb 2021 19:04:02 +0200 Subject: [PATCH 7/8] Changed changeset bump to be patch --- .changeset/moody-mice-cheer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/moody-mice-cheer.md b/.changeset/moody-mice-cheer.md index f2e5d29ba1..50d0e57b58 100644 --- a/.changeset/moody-mice-cheer.md +++ b/.changeset/moody-mice-cheer.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Added an option to scan GitHub for repositories using a new location type `github-discovery`. From d444fee636279c557b8570852adb05e57b776ade Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Tue, 9 Feb 2021 20:32:15 +0200 Subject: [PATCH 8/8] Fixed parseUrl to output catalogPaths beginning with '/' --- .../src/ingestion/processors/GithubDiscoveryProcessor.test.ts | 4 ++-- .../src/ingestion/processors/GithubDiscoveryProcessor.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index fe6bb51521..41698e1d04 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -33,14 +33,14 @@ describe('GithubDiscoveryProcessor', () => { ).toEqual({ org: 'foo', repoSearchPath: /^proj$/, - catalogPath: 'blob/master/catalog.yaml', + catalogPath: '/blob/master/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), ).toEqual({ org: 'foo', repoSearchPath: /^proj.*$/, - catalogPath: 'blob/master/catalog.yaml', + catalogPath: '/blob/master/catalog.yaml', }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index f96943eebb..9af247fd25 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -92,7 +92,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { results.location( { type: 'url', - target: `${repository.url}/${catalogPath}`, + target: `${repository.url}${catalogPath}`, }, false, ), @@ -118,7 +118,7 @@ export function parseUrl( return { org: decodeURIComponent(path[0]), repoSearchPath: escapeRegExp(decodeURIComponent(path[1])), - catalogPath: decodeURIComponent(path.slice(2).join('/')), + catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`, }; }