diff --git a/CHANGELOG.md b/CHANGELOG.md index 294c57815e..814716b60c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re - Change root `tsc` output dir to `dist-types`, in order to allow for standalone plugin repos. [#2278](https://github.com/spotify/backstage/pull/2278) +### @backstage/catalog-backend + +- We have simplified the way that GitHub ingestion works. The `catalog.processors.githubApi` key is deprecated, in favor of `catalog.processors.github`. At the same time, the location type `github/api` is likewise deprecated, in favor of `github`. This location type now serves both raw HTTP reads and APIv3 reads, depending on how you configure it. It also supports having several providers at once - for example, both public GitHub and an internal GitHub Enterprise, with different keys. If you still use the `catalog.processors.githubApi` config key, things will work but you will get a deprecation warning at startup. In a later release, support for the old key will go away entirely. See the [configuration section in the docs](https://backstage.io/docs/features/software-catalog/configuration) for more details. + ## v0.1.1-alpha.21 - Added many more frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084) diff --git a/app-config.yaml b/app-config.yaml index 5ca76ce789..60ecb813f7 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -60,21 +60,23 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: $secret: env: GITHUB_PRIVATE_TOKEN - # Example for how to add your GitHub Enterprise instance: + #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 # token: # $secret: # env: GHE_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - target: https://ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 0bcfaabe0a..c98bd73ca3 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -4,6 +4,67 @@ title: Catalog Configuration description: Documentation on Software Catalog Configuration --- +## Processors + +The catalog makes use of so called processors to perform all kinds of ingestion +tasks, such as reading raw entity data from a remote source, parsing it, +transforming it, and validating it. These processors are configured under the +`catalog.processors` key. + +### Processor: github + +The `github` processor is responsible for fetching entity data from files on +GitHub or GitHub Enterprise. The configuration for this processor lives under +`catalog.processors.github`. Example: + +```yaml +catalog: + processors: + github: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + - target: https://ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_PRIVATE_TOKEN +``` + +The main subkey is `providers`, where you can list the various GitHub compatible +providers you want to be able to fetch data from. Each entry is a structure with +up to four elements: + +- `target` (required): The string prefix of the location target that you want to + match on, with no trailing slash. For GitHub, it should be exactly + `https://github.com`. +- `token` (optional): An authentication token as expected by GitHub. If + supplied, it will be passed along with all calls to this provider, both API + and raw. If it is not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is github, you can leave it out + to be inferred automatically. For a GitHub Enterprise installation, it is + commonly at `https://api.` or `https:///api/v3`. +- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is public GitHub, you can leave + it out to be inferred automatically. For a GitHub Enterprise installation, it + is commonly at `https://api.` or `https:///api/v3`. + +You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for +public GitHub, for which we can infer them). The `apiBaseUrl` will always be +preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be +preferred. + +If you do not supply a public GitHub provider, one will be added automatically, +silently at startup for convenience. So you only have to list it if you want to +supply a token for it - and if you do, you can also leave out the `apiBaseUrl` +and `rawBaseUrl` fields. + ## Static Location Configuration To enable declarative catalog setups, it is possible to add locations to the diff --git a/microsite/sidebars.json b/microsite/sidebars.json index aac524c538..8fa4935a38 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -40,6 +40,7 @@ "ids": [ "features/software-catalog/software-catalog-overview", "features/software-catalog/installation", + "features/software-catalog/configuration", "features/software-catalog/system-model", "features/software-catalog/descriptor-format", "features/software-catalog/well-known-annotations", diff --git a/mkdocs.yml b/mkdocs.yml index a74ef5737f..ba607f8a6e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,10 +28,11 @@ nav: - Features: - Software Catalog: - Overview: 'features/software-catalog/index.md' + - Installation: 'features/software-catalog/installation.md' + - Configuration: 'features/software-catalog/configuration.md' - System model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' - Well-known Annotations: 'features/software-catalog/well-known-annotations.md' - - Configuration: 'features/software-catalog/configuration.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index d8d5adae37..a7d7aa8010 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -67,10 +67,6 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index d1711cf3d5..bb62efdebb 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -27,6 +27,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", + "git-url-parse": "^11.2.0", "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", @@ -40,6 +41,7 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.21", + "@types/git-url-parse": "^9.0.0", "@types/lodash": "^4.14.151", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 6f3a27f00d..8da6e20ab3 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -27,7 +27,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; @@ -67,18 +66,19 @@ export class LocationReaders implements LocationReader { private readonly rulesEnforcer: CatalogRulesEnforcer; static defaultProcessors(options: { + logger: Logger; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { + logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GithubReaderProcessor(config), - GithubApiReaderProcessor.fromConfig(config), + GithubReaderProcessor.fromConfig(config, logger), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), @@ -94,7 +94,7 @@ export class LocationReaders implements LocationReader { constructor({ logger = getVoidLogger(), config, - processors = LocationReaders.defaultProcessors({ config }), + processors = LocationReaders.defaultProcessors({ logger, config }), }: Options) { this.logger = logger; this.processors = processors; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts deleted file mode 100644 index 48f38f0e97..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { ConfigReader } from '@backstage/config'; -import { - getRawUrl, - getRequestOptions, - GithubApiReaderProcessor, - ProviderConfig, - readConfig, -} from './GithubApiReaderProcessor'; - -describe('GithubApiReaderProcessor', () => { - describe('getRequestOptions', () => { - it('sets the correct API version', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; - expect((getRequestOptions(config).headers as any).Accept).toEqual( - 'application/vnd.github.v3.raw', - ); - }); - - it('inserts a token when needed', () => { - const withToken: ProviderConfig = { - target: '', - apiBaseUrl: '', - token: 'A', - }; - const withoutToken: ProviderConfig = { - target: '', - apiBaseUrl: '', - }; - expect( - (getRequestOptions(withToken).headers as any).Authorization, - ).toEqual('token A'); - expect( - (getRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - }); - - describe('getRawUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; - expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - - it('passes through the happy path', () => { - const config: ProviderConfig = { - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - }; - expect( - getRawUrl( - 'https://github.com/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', - ), - ); - }); - }); - - describe('readConfig', () => { - function config( - providers: { target: string; apiBaseUrl?: string; token?: string }[], - ) { - return ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { processors: { githubApi: { providers } } }, - }, - }, - ]); - } - - it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([])); - expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - }); - - it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig(config([{ target: 'https://github.com' }])); - expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - }); - - it('rejects custom targets with no API base URL', () => { - expect(() => - readConfig(config([{ target: 'https://ghe.company.com' }])), - ).toThrow( - 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl', - ); - }); - - it('rejects funky configs', () => { - expect(() => readConfig(config([{ target: 7 } as any]))).toThrow( - /target/, - ); - expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow( - /target/, - ); - expect(() => - readConfig( - config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), - ), - ).toThrow(/apiBaseUrl/); - expect(() => - readConfig(config([{ target: 'https://github.com', token: 7 } as any])), - ).toThrow(/token/); - }); - }); - - describe('implementation', () => { - it('rejects unknown types', async () => { - const processor = new GithubApiReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'not-github/api', - target: 'https://github.com', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).resolves.toBeFalsy(); - }); - - it('rejects unknown targets', async () => { - const processor = new GithubApiReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'github/api', - target: 'https://not.github.com/apa', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).rejects.toThrow( - /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, - ); - }); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts deleted file mode 100644 index 812bf6f488..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import fetch, { HeadersInit, RequestInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -/** - * The configuration parameters for a single GitHub API provider. - */ -export type ProviderConfig = { - /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. - */ - target: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.github.com", - * with no trailing slash. - */ - apiBaseUrl: string; - - /** - * The authorization token to use for requests to this provider. - * - * If no token is specified, anonymous API access is used. - */ - token?: string; -}; - -export function getRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/path/to/c.yaml -// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getRawUrl(target: string, provider: ProviderConfig): URL { - try { - const oldPath = new URL(target).pathname.split('/'); - const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath; - - if ( - !userOrOrg || - !repoName || - (blobOrRaw !== 'blob' && blobOrRaw !== 'raw') || - !restOfPath.join('/').match(/\.ya?ml$/) - ) { - throw new Error('Wrong URL or Invalid file path'); - } - - // Transform to API path - const newPath = [ - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -export function readConfig(configRoot: Config): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. - const config = configRoot.getOptionalConfig('catalog.processors.githubApi'); - const providerConfigs = config?.getOptionalConfigArray('providers') ?? []; - const legacyToken = config?.getOptionalString('privateToken'); - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - const token = providerConfig.getOptionalString('token'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { - apiBaseUrl = 'https://api.github.com'; - } else { - throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl`, - ); - } - - providers.push({ target, apiBaseUrl, token }); - } - - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { - providers.push({ - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - token: legacyToken, - }); - } - - return providers; -} - -/** - * A processor that adds the ability to read files from GitHub v3 APIs, such as - * the one exposed by GitHub itself. - */ -export class GithubApiReaderProcessor implements LocationProcessor { - private providers: ProviderConfig[]; - - static fromConfig(config: Config) { - return new GithubApiReaderProcessor(readConfig(config)); - } - - constructor(providers: ProviderConfig[]) { - this.providers = providers; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'github/api') { - return false; - } - - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`, - ); - } - - try { - const url = getRawUrl(location.target, provider); - const options = getRequestOptions(provider); - const response = await fetch(url.toString(), options); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts new file mode 100644 index 0000000000..3ab24541e7 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -0,0 +1,269 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { + getApiRequestOptions, + getApiUrl, + getRawRequestOptions, + getRawUrl, + GithubReaderProcessor, + ProviderConfig, + readConfig, +} from './GithubReaderProcessor'; + +describe('GithubReaderProcessor', () => { + describe('getApiRequestOptions', () => { + it('sets the correct API version', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect((getApiRequestOptions(config).headers as any).Accept).toEqual( + 'application/vnd.github.v3.raw', + ); + }); + + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + }; + expect( + (getApiRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getApiRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getRawRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + }; + expect( + (getRawRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getRawRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getApiUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getApiUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + apiBaseUrl: 'https://ghe.mycompany.net/api/v3', + }; + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + }); + + describe('getRawUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }; + expect( + getRawUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + rawBaseUrl: 'https://ghe.mycompany.net/raw', + }; + expect( + getRawUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'), + ); + }); + }); + + describe('readConfig', () => { + function config( + providers: { target: string; apiBaseUrl?: string; token?: string }[], + ) { + return ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { processors: { github: { providers } } }, + }, + }, + ]); + } + + it('adds a default GitHub entry when missing', () => { + const output = readConfig(config([]), getVoidLogger()); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('injects the correct GitHub API base URL when missing', () => { + const output = readConfig( + config([{ target: 'https://github.com' }]), + getVoidLogger(), + ); + expect(output).toEqual([ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); + + it('rejects custom targets with no base URLs', () => { + expect(() => + readConfig( + config([{ target: 'https://ghe.company.com' }]), + getVoidLogger(), + ), + ).toThrow( + 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', + ); + }); + + it('rejects funky configs', () => { + expect(() => + readConfig(config([{ target: 7 } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig( + config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/apiBaseUrl/); + expect(() => + readConfig( + config([{ target: 'https://github.com', token: 7 } as any]), + getVoidLogger(), + ), + ).toThrow(/token/); + }); + }); + + describe('implementation', () => { + it('rejects unknown types', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'not-github/api', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'github/api', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index 9f38d782fe..4f0c66148f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,33 +15,207 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -import { Config } from '@backstage/config'; -export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string; +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; - constructor(config?: Config) { - this.privateToken = - config?.getOptionalString('catalog.processors.github.privateToken') ?? ''; + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; } - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; + return { + headers, + }; +} - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); } - const requestOptions: RequestInit = { - headers, - }; + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} - return requestOptions; +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; } async readLocation( @@ -49,16 +223,30 @@ export class GithubReaderProcessor implements LocationProcessor { optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'github') { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { return false; } - try { - const url = this.buildRawUrl(location.target); + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } - // TODO(freben): Should "hard" errors thrown by this line be treated as - // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + const response = await fetch(url.toString(), options); if (response.ok) { const data = await response.buffer(); @@ -80,41 +268,4 @@ export class GithubReaderProcessor implements LocationProcessor { return true; } - - // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } }