diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index b2db27770b..07fd000618 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -34,6 +34,7 @@ "@backstage/backend-common": "^0.5.0", "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", + "@backstage/integration": "^0.3.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts index a3713fb374..9daecb26d3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts @@ -56,12 +56,15 @@ describe('GithubOrgReaderProcessor', () => { describe('reject unrelated entries', () => { it('rejects unknown types', async () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + ]), logger: getVoidLogger(), }); const location: LocationSpec = { @@ -75,12 +78,22 @@ describe('GithubOrgReaderProcessor', () => { it('rejects unknown targets', async () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + [ + 'ghe.example.net', + { + host: 'ghe.example.net', + apiBaseUrl: 'https://ghe.example.net/api/v3', + }, + ], + ]), logger: getVoidLogger(), }); const location: LocationSpec = { @@ -90,19 +103,22 @@ describe('GithubOrgReaderProcessor', () => { await expect( processor.readLocation(location, false, () => {}), ).rejects.toThrow( - /There is no GitHub Org provider that matches https:\/\/not.github.com\/apa/, + /There is no GitHub integration that matches https:\/\/not.github.com\/apa/, ); }); }); describe('handles repositories', () => { const processor = new GithubDiscoveryProcessor({ - providers: [ - { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }, - ], + gitHubConfigMap: new Map([ + [ + 'github.com', + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + ]), logger: getVoidLogger(), }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts index e5590b6179..698f0ad987 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts @@ -16,32 +16,41 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { - getOrganizationRepositories, - ProviderConfig, - readGithubConfig, -} from './github'; +import { getOrganizationRepositories } from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** - * Extracts teams and users out of a GitHub org. + * Extracts repositories out of a GitHub org. */ export class GithubDiscoveryProcessor implements CatalogProcessor { - private readonly providers: ProviderConfig[]; + private readonly gitHubConfigMap: Map; private readonly logger: Logger; static fromConfig(config: Config, options: { logger: Logger }) { + const configs = readGitHubIntegrationConfigs( + config.getOptionalConfigArray('integrations.github') ?? [], + ); + const gitHubConfigMap = new Map(configs.map(c => [c.host, c])); + return new GithubDiscoveryProcessor({ ...options, - providers: readGithubConfig(config), + gitHubConfigMap, }); } - constructor(options: { providers: ProviderConfig[]; logger: Logger }) { - this.providers = options.providers; + constructor(options: { + gitHubConfigMap: Map; + logger: Logger; + }) { + this.gitHubConfigMap = options.gitHubConfigMap; this.logger = options.logger; } @@ -54,24 +63,23 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { return false; } - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), + const gitHubConfig = this.gitHubConfigMap.get( + new URL(location.target).hostname, ); - if (!provider) { + if (!gitHubConfig) { throw new Error( - `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`, ); } - + const { headers } = await GithubCredentialsProvider.create( + gitHubConfig, + ).getCredentials({ url: location.target }); const { org, repoSearchPath, catalogPath } = parseUrl(location.target); - const client = !provider.token - ? graphql - : graphql.defaults({ - baseUrl: provider.apiBaseUrl, - headers: { - authorization: `token ${provider.token}`, - }, - }); + + const client = graphql.defaults({ + baseUrl: gitHubConfig.apiBaseUrl, + headers, + }); // Read out all of the raw data const startTimestamp = Date.now(); diff --git a/yarn.lock b/yarn.lock index 7c743bc5b3..810090a45f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2473,9 +2473,11 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.7.0" + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.2.0.tgz#e3fe2a4ddeb6a9b6ec480c80cb2b9c39cb245576" + integrity sha512-Y1ocdRpBlxK/VrJQjHlQd0bgADECd1B2NRjwd8ss46ibT5hwLvMOfD80+Fa7oPLu0ktJrH4lq0pNIIJIml48zA== dependencies: - "@backstage/config" "^0.1.2" + "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2484,9 +2486,11 @@ yup "^0.29.3" "@backstage/catalog-model@^0.3.0": - version "0.7.0" + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" + integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== dependencies: - "@backstage/config" "^0.1.2" + "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" "@types/yup" "^0.29.8" json-schema "^0.2.5" @@ -2495,16 +2499,17 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.5.0" + version "0.3.2" + resolved "https://registry.npmjs.org/@backstage/core/-/core-0.3.2.tgz#a8209126d5076cf4a8b9bd632fe4e5e2edb62916" + integrity sha512-i5d+Wh8js4qEWoAsPY5L7HVSWpumr1OhfF2dUCGYdyW6AMqVJPca6+n6zp1Rg2CO+J9norp44XAVVCbyhtUpig== dependencies: - "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.8" - "@backstage/theme" "^0.2.2" + "@backstage/config" "^0.1.1" + "@backstage/core-api" "^0.2.1" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" "@types/dagre" "^0.7.44" - "@types/prop-types" "^15.7.3" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" @@ -2513,7 +2518,7 @@ d3-shape "^2.0.0" d3-zoom "^2.0.0" dagre "^0.8.5" - immer "^8.0.1" + immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" @@ -2532,6 +2537,18 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" +"@backstage/integration@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/integration/-/integration-0.3.1.tgz#4bdf1883b7149f7d4866bf1732d8e371f37ed143" + integrity sha512-aHfZ83z3XmO3R1SPG3r3W9KjSab/aFJ+51LjvRr6OCthfFTIusEmFq4cro5RUn7IPTn/0L8qpBSW2xH6FlRuVw== + dependencies: + "@backstage/config" "^0.1.2" + "@octokit/auth-app" "^2.10.5" + "@octokit/rest" "^18.0.12" + cross-fetch "^3.0.6" + git-url-parse "^11.4.4" + luxon "^1.25.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -14180,6 +14197,13 @@ git-url-parse@^11.4.3: dependencies: git-up "^4.0.0" +git-url-parse@^11.4.4: + version "11.4.4" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.4.tgz#5d747debc2469c17bc385719f7d0427802d83d77" + integrity sha512-Y4o9o7vQngQDIU9IjyCmRJBin5iYjI5u9ZITnddRZpD7dcCFQj2sL2XuMNbLRE4b4B/4ENPsp2Q8P44fjAZ0Pw== + dependencies: + git-up "^4.0.0" + gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" @@ -15309,6 +15333,11 @@ immer@1.10.0: resolved "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" integrity sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg== +immer@^7.0.9: + version "7.0.15" + resolved "https://registry.npmjs.org/immer/-/immer-7.0.15.tgz#dc3bc6db87401659d2e737c67a21b227c484a4ad" + integrity sha512-yM7jo9+hvYgvdCQdqvhCNRRio0SCXc8xDPzA25SvKWa7b1WVPjLwQs1VYU5JPXjcJPTqAa5NP5dqpORGYBQ2AA== + immer@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656"