From df5e7d4e43f2562fb72f463376025c89116f2abc Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 4 Feb 2021 17:40:35 +0200 Subject: [PATCH] Fixes following CR --- .../GithubDiscoveryProcessor.test.ts | 110 ++++++++++-------- .../processors/GithubDiscoveryProcessor.ts | 28 ++--- yarn.lock | 30 ++--- 3 files changed, 83 insertions(+), 85 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index 9daecb26d3..fe6bb51521 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -18,27 +18,28 @@ import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; import { getOrganizationRepositories } from './github'; +import { ConfigReader } from '@backstage/config'; jest.mock('./github'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories >; -describe('GithubOrgReaderProcessor', () => { +describe('GithubDiscoveryProcessor', () => { describe('parseUrl', () => { it('parses well formed URLs', () => { expect( parseUrl('https://github.com/foo/proj/blob/master/catalog.yaml'), ).toEqual({ org: 'foo', - repoSearchPath: /proj/, + repoSearchPath: /^proj$/, catalogPath: 'blob/master/catalog.yaml', }); expect( parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'), ).toEqual({ org: 'foo', - repoSearchPath: /proj.*/, + repoSearchPath: /^proj.*$/, catalogPath: 'blob/master/catalog.yaml', }); }); @@ -55,18 +56,14 @@ describe('GithubOrgReaderProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); const location: LocationSpec = { type: 'not-github-discovery', target: 'https://github.com', @@ -77,25 +74,17 @@ describe('GithubOrgReaderProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - [ - 'ghe.example.net', - { - host: 'ghe.example.net', - apiBaseUrl: 'https://ghe.example.net/api/v3', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'blob' }, + { host: 'ghe.example.net', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); const location: LocationSpec = { type: 'github-discovery', target: 'https://not.github.com/apa', @@ -109,18 +98,14 @@ describe('GithubOrgReaderProcessor', () => { }); describe('handles repositories', () => { - const processor = new GithubDiscoveryProcessor({ - gitHubConfigMap: new Map([ - [ - 'github.com', - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], - ]), - logger: getVoidLogger(), - }); + const processor = GithubDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); beforeEach(() => { mockGetOrganizationRepositories.mockClear(); @@ -160,7 +145,7 @@ describe('GithubOrgReaderProcessor', () => { }); }); - it('filter unrelated repositories', async () => { + it('output repositories with wildcards', async () => { const location: LocationSpec = { type: 'github-discovery', target: @@ -202,5 +187,36 @@ describe('GithubOrgReaderProcessor', () => { optional: false, }); }); + it('filter unrelated repositories', async () => { + const location: LocationSpec = { + type: 'github-discovery', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }; + mockGetOrganizationRepositories.mockResolvedValueOnce({ + repositories: [ + { name: 'abstest', url: 'https://github.com/backstage/abctest' }, + { + name: 'test', + url: 'https://github.com/backstage/test', + }, + { + name: 'testxyz', + url: 'https://github.com/backstage/testxyz', + }, + ], + }); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: 'https://github.com/backstage/test/blob/master/catalog.yaml', + }, + optional: false, + }); + }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index 698f0ad987..f96943eebb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -18,8 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { GithubCredentialsProvider, - GitHubIntegrationConfig, - readGitHubIntegrationConfigs, + ScmIntegrations, } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; @@ -31,26 +30,20 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; * Extracts repositories out of a GitHub org. */ export class GithubDiscoveryProcessor implements CatalogProcessor { - private readonly gitHubConfigMap: Map; + private readonly integrations: ScmIntegrations; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { - const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [], - ); - const gitHubConfigMap = new Map(configs.map(c => [c.host, c])); + const integrations = ScmIntegrations.fromConfig(config); return new GithubDiscoveryProcessor({ ...options, - gitHubConfigMap, + integrations, }); } - constructor(options: { - gitHubConfigMap: Map; - logger: Logger; - }) { - this.gitHubConfigMap = options.gitHubConfigMap; + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; this.logger = options.logger; } @@ -63,9 +56,8 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { return false; } - const gitHubConfig = this.gitHubConfigMap.get( - new URL(location.target).hostname, - ); + const gitHubConfig = this.integrations.github.byUrl(location.target) + ?.config; if (!gitHubConfig) { throw new Error( `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, @@ -83,7 +75,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { // Read out all of the raw data const startTimestamp = Date.now(); - this.logger.info('Reading GitHub repositories'); + this.logger.info(`Reading GitHub repositories from ${location.target}`); const { repositories } = await getOrganizationRepositories(client, org); @@ -134,5 +126,5 @@ export function parseUrl( } export function escapeRegExp(str: string): RegExp { - return new RegExp(str.replace(/\*/g, '.*')); + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); } diff --git a/yarn.lock b/yarn.lock index 810090a45f..c20a8c505d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2473,11 +2473,9 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" - integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== + version "0.7.0" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2486,11 +2484,9 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" - integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== + version "0.7.0" dependencies: - "@backstage/config" "^0.1.1" + "@backstage/config" "^0.1.2" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2499,17 +2495,16 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.3.2" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.3.2.tgz#a8209126d5076cf4a8b9bd632fe4e5e2edb62916" - integrity sha512-i5d+Wh8js4qEWoAsPY5L7HVSWpumr1OhfF2dUCGYdyW6AMqVJPca6+n6zp1Rg2CO+J9norp44XAVVCbyhtUpig== + version "0.5.0" dependencies: - "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" + "@backstage/config" "^0.1.2" + "@backstage/core-api" "^0.2.8" + "@backstage/theme" "^0.2.2" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" "@types/dagre" "^0.7.44" + "@types/prop-types" "^15.7.3" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" @@ -2518,7 +2513,7 @@ d3-shape "^2.0.0" d3-zoom "^2.0.0" dagre "^0.8.5" - immer "^7.0.9" + immer "^8.0.1" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" @@ -15333,11 +15328,6 @@ immer@1.10.0: resolved "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" integrity sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg== -immer@^7.0.9: - version "7.0.15" - resolved "https://registry.npmjs.org/immer/-/immer-7.0.15.tgz#dc3bc6db87401659d2e737c67a21b227c484a4ad" - integrity sha512-yM7jo9+hvYgvdCQdqvhCNRRio0SCXc8xDPzA25SvKWa7b1WVPjLwQs1VYU5JPXjcJPTqAa5NP5dqpORGYBQ2AA== - immer@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656"