diff --git a/app-config.yaml b/app-config.yaml index 05c19f75a3..3349a6585f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -62,9 +62,17 @@ catalog: $secret: env: GITHUB_PRIVATE_TOKEN githubApi: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + # Example for how to add your GitHub Enterprise instance: + # - target: https://ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 7282b732b4..5fae2e4592 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -55,6 +55,25 @@ scaffolder: visibility: public # or 'internal' or 'private' catalog: + rules: + - allow: [Component, API, Group, Template, Location] + processors: + github: + privateToken: + $secret: + env: GITHUB_PRIVATE_TOKEN + githubApi: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + # Example for how to add your GitHub Enterprise instance: + # - target: https://ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN locations: # Backstage example components - type: github diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a86e073af9..6f3a27f00d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -78,7 +78,7 @@ export class LocationReaders implements LocationReader { StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), new GithubReaderProcessor(config), - new GithubApiReaderProcessor(config), + GithubApiReaderProcessor.fromConfig(config), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index 51c3cf4cd0..48f38f0e97 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -14,116 +14,149 @@ * limitations under the License. */ -import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; +import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import { + getRawUrl, + getRequestOptions, + GithubApiReaderProcessor, + ProviderConfig, + readConfig, +} from './GithubApiReaderProcessor'; describe('GithubApiReaderProcessor', () => { - const createConfig = (token: string | undefined) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { - processors: { - githubApi: { - privateToken: token, - }, - }, - }, - }, - }, - ]); + describe('getRequestOptions', () => { + it('sets the correct API version', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect((getRequestOptions(config).headers as any).Accept).toEqual( + 'application/vnd.github.v3.raw', + ); + }); - it('should build raw api', () => { - const processor = new GithubApiReaderProcessor(createConfig(undefined)); - - const tests = [ - { - target: 'https://github.com/a/b/blob/master/path/to/c.yaml', - url: new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - { - target: - 'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml', - url: new URL( - 'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master', - ), - err: undefined, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + apiBaseUrl: '', + }; + expect( + (getRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); }); - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - Accept: 'application/vnd.github.v3.raw', - Authorization: 'token 0123456789', - }, - }, - }, - { - token: '', - err: - "Invalid type in config for key 'catalog.processors.githubApi.privateToken' in '', got empty-string, wanted string", - expect: { - headers: { - Accept: 'application/vnd.github.v3.raw', - }, - }, - }, - { - token: undefined, - expect: { - headers: { - Accept: 'application/vnd.github.v3.raw', - }, - }, - }, - ]; + describe('getRawUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); - for (const test of tests) { - if (test.err) { - expect( - () => new GithubApiReaderProcessor(createConfig(test.token)), - ).toThrowError(test.err); - } else { - const processor = new GithubApiReaderProcessor( - createConfig(test.token), - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } + it('passes through the happy path', () => { + const config: ProviderConfig = { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getRawUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + }); + + describe('readConfig', () => { + function config( + providers: { target: string; apiBaseUrl?: string; token?: string }[], + ) { + return ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { processors: { githubApi: { providers } } }, + }, + }, + ]); } + + it('adds a default GitHub entry when missing', () => { + const output = readConfig(config([])); + expect(output).toEqual([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + }); + + it('injects the correct GitHub API base URL when missing', () => { + const output = readConfig(config([{ target: 'https://github.com' }])); + expect(output).toEqual([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + }); + + it('rejects custom targets with no API base URL', () => { + expect(() => + readConfig(config([{ target: 'https://ghe.company.com' }])), + ).toThrow( + 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl', + ); + }); + + it('rejects funky configs', () => { + expect(() => readConfig(config([{ target: 7 } as any]))).toThrow( + /target/, + ); + expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow( + /target/, + ); + expect(() => + readConfig( + config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), + ), + ).toThrow(/apiBaseUrl/); + expect(() => + readConfig(config([{ target: 'https://github.com', token: 7 } as any])), + ).toThrow(/token/); + }); + }); + + describe('implementation', () => { + it('rejects unknown types', async () => { + const processor = new GithubApiReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'not-github/api', + target: 'https://github.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + + it('rejects unknown targets', async () => { + const processor = new GithubApiReaderProcessor([ + { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + ]); + const location: LocationSpec = { + type: 'github/api', + target: 'https://not.github.com/apa', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + ); + }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts index f8d1d6caf8..812bf6f488 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -15,34 +15,135 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import { Config } from '@backstage/config'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -import { Config } from '@backstage/config'; -export class GithubApiReaderProcessor implements LocationProcessor { - private privateToken: string; +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; - constructor(config: Config) { - this.privateToken = - config.getOptionalString('catalog.processors.githubApi.privateToken') ?? - ''; + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + */ + apiBaseUrl: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous API access is used. + */ + token?: string; +}; + +export function getRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; } - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; + return { + headers, + }; +} - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const oldPath = new URL(target).pathname.split('/'); + const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath; + + if ( + !userOrOrg || + !repoName || + (blobOrRaw !== 'blob' && blobOrRaw !== 'raw') || + !restOfPath.join('/').match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or Invalid file path'); } - const requestOptions: RequestInit = { - headers, - }; + // Transform to API path + const newPath = [ + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} - return requestOptions; +export function readConfig(configRoot: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const config = configRoot.getOptionalConfig('catalog.processors.githubApi'); + const providerConfigs = config?.getOptionalConfigArray('providers') ?? []; + const legacyToken = config?.getOptionalString('privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } else { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubApiReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config) { + return new GithubApiReaderProcessor(readConfig(config)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; } async readLocation( @@ -54,10 +155,19 @@ export class GithubApiReaderProcessor implements LocationProcessor { return false; } - try { - const url = this.buildRawUrl(location.target); + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`, + ); + } - const response = await fetch(url.toString(), this.getRequestOptions()); + try { + const url = getRawUrl(location.target, provider); + const options = getRequestOptions(provider); + const response = await fetch(url.toString(), options); if (response.ok) { const data = await response.buffer(); @@ -79,50 +189,4 @@ export class GithubApiReaderProcessor implements LocationProcessor { return true; } - - // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ref, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL or Invalid file path'); - } - - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; - url.protocol = 'https'; - url.search = `ref=${ref}`; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } }