diff --git a/CHANGELOG.md b/CHANGELOG.md index 294c57815e..814716b60c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re - Change root `tsc` output dir to `dist-types`, in order to allow for standalone plugin repos. [#2278](https://github.com/spotify/backstage/pull/2278) +### @backstage/catalog-backend + +- We have simplified the way that GitHub ingestion works. The `catalog.processors.githubApi` key is deprecated, in favor of `catalog.processors.github`. At the same time, the location type `github/api` is likewise deprecated, in favor of `github`. This location type now serves both raw HTTP reads and APIv3 reads, depending on how you configure it. It also supports having several providers at once - for example, both public GitHub and an internal GitHub Enterprise, with different keys. If you still use the `catalog.processors.githubApi` config key, things will work but you will get a deprecation warning at startup. In a later release, support for the old key will go away entirely. See the [configuration section in the docs](https://backstage.io/docs/features/software-catalog/configuration) for more details. + ## v0.1.1-alpha.21 - Added many more frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 0bcfaabe0a..c98bd73ca3 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -4,6 +4,67 @@ title: Catalog Configuration description: Documentation on Software Catalog Configuration --- +## Processors + +The catalog makes use of so called processors to perform all kinds of ingestion +tasks, such as reading raw entity data from a remote source, parsing it, +transforming it, and validating it. These processors are configured under the +`catalog.processors` key. + +### Processor: github + +The `github` processor is responsible for fetching entity data from files on +GitHub or GitHub Enterprise. The configuration for this processor lives under +`catalog.processors.github`. Example: + +```yaml +catalog: + processors: + github: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + - target: https://ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_PRIVATE_TOKEN +``` + +The main subkey is `providers`, where you can list the various GitHub compatible +providers you want to be able to fetch data from. Each entry is a structure with +up to four elements: + +- `target` (required): The string prefix of the location target that you want to + match on, with no trailing slash. For GitHub, it should be exactly + `https://github.com`. +- `token` (optional): An authentication token as expected by GitHub. If + supplied, it will be passed along with all calls to this provider, both API + and raw. If it is not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is github, you can leave it out + to be inferred automatically. For a GitHub Enterprise installation, it is + commonly at `https://api.` or `https:///api/v3`. +- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is public GitHub, you can leave + it out to be inferred automatically. For a GitHub Enterprise installation, it + is commonly at `https://api.` or `https:///api/v3`. + +You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for +public GitHub, for which we can infer them). The `apiBaseUrl` will always be +preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be +preferred. + +If you do not supply a public GitHub provider, one will be added automatically, +silently at startup for convenience. So you only have to list it if you want to +supply a token for it - and if you do, you can also leave out the `apiBaseUrl` +and `rawBaseUrl` fields. + ## Static Location Configuration To enable declarative catalog setups, it is possible to add locations to the diff --git a/microsite/sidebars.json b/microsite/sidebars.json index f7aa0630ef..f3c2cf7e76 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -40,6 +40,7 @@ "ids": [ "features/software-catalog/software-catalog-overview", "features/software-catalog/installation", + "features/software-catalog/configuration", "features/software-catalog/system-model", "features/software-catalog/descriptor-format", "features/software-catalog/well-known-annotations", diff --git a/mkdocs.yml b/mkdocs.yml index a74ef5737f..ba607f8a6e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,10 +28,11 @@ nav: - Features: - Software Catalog: - Overview: 'features/software-catalog/index.md' + - Installation: 'features/software-catalog/installation.md' + - Configuration: 'features/software-catalog/configuration.md' - System model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' - Well-known Annotations: 'features/software-catalog/well-known-annotations.md' - - Configuration: 'features/software-catalog/configuration.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a564e9725c..8da6e20ab3 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -66,17 +66,19 @@ export class LocationReaders implements LocationReader { private readonly rulesEnforcer: CatalogRulesEnforcer; static defaultProcessors(options: { + logger: Logger; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { + logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - GithubReaderProcessor.fromConfig(config), + GithubReaderProcessor.fromConfig(config, logger), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), @@ -92,7 +94,7 @@ export class LocationReaders implements LocationReader { constructor({ logger = getVoidLogger(), config, - processors = LocationReaders.defaultProcessors({ config }), + processors = LocationReaders.defaultProcessors({ logger, config }), }: Options) { this.logger = logger; this.processors = processors; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index e3c7611e80..3ab24541e7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -14,13 +14,14 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { - getApiUrl, getApiRequestOptions, - getRawUrl, + getApiUrl, getRawRequestOptions, + getRawUrl, GithubReaderProcessor, ProviderConfig, readConfig, @@ -179,7 +180,7 @@ describe('GithubReaderProcessor', () => { } it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([])); + const output = readConfig(config([]), getVoidLogger()); expect(output).toEqual([ { target: 'https://github.com', @@ -190,7 +191,10 @@ describe('GithubReaderProcessor', () => { }); it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig(config([{ target: 'https://github.com' }])); + const output = readConfig( + config([{ target: 'https://github.com' }]), + getVoidLogger(), + ); expect(output).toEqual([ { target: 'https://github.com', @@ -202,26 +206,33 @@ describe('GithubReaderProcessor', () => { it('rejects custom targets with no base URLs', () => { expect(() => - readConfig(config([{ target: 'https://ghe.company.com' }])), + readConfig( + config([{ target: 'https://ghe.company.com' }]), + getVoidLogger(), + ), ).toThrow( 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', ); }); it('rejects funky configs', () => { - expect(() => readConfig(config([{ target: 7 } as any]))).toThrow( - /target/, - ); - expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow( - /target/, - ); + expect(() => + readConfig(config([{ target: 7 } as any]), getVoidLogger()), + ).toThrow(/target/); + expect(() => + readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), + ).toThrow(/target/); expect(() => readConfig( config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), + getVoidLogger(), ), ).toThrow(/apiBaseUrl/); expect(() => - readConfig(config([{ target: 'https://github.com', token: 7 } as any])), + readConfig( + config([{ target: 'https://github.com', token: 7 } as any]), + getVoidLogger(), + ), ).toThrow(/token/); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index f83469bd16..4f0c66148f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; @@ -139,9 +140,16 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL { } } -export function readConfig(config: Config): ProviderConfig[] { +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { const providers: ProviderConfig[] = []; + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + // In a previous version of the configuration, we only supported github, // and the "privateToken" key held the token to use for it. The new // configuration method is to use the "providers" key instead. @@ -202,8 +210,8 @@ export function readConfig(config: Config): ProviderConfig[] { export class GithubReaderProcessor implements LocationProcessor { private providers: ProviderConfig[]; - static fromConfig(config: Config) { - return new GithubReaderProcessor(readConfig(config)); + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); } constructor(providers: ProviderConfig[]) { @@ -238,7 +246,6 @@ export class GithubReaderProcessor implements LocationProcessor { const options = useApi ? getApiRequestOptions(provider) : getRawRequestOptions(provider); - console.log(url.toString()); const response = await fetch(url.toString(), options); if (response.ok) {