diff --git a/app-config.yaml b/app-config.yaml index d4464332fa..d21e858e8a 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -67,53 +67,42 @@ kubernetes: clusterLocatorMethod: 'configMultiTenant' clusters: [] -# This sections configures the URL readers used by various parts of -# Backstage to read data from remote URLs -readers: - byHost: - - type: github - host: github.com - config: - token: - $secret: - env: GITHUB_PRIVATE_TOKEN +integrations: + github: + - host: github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN ### Example for how to add your GitHub Enterprise instance using the API: - # - type: github - # host: ghe.example.net - # config: - # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN + # - host: ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): - # - type: github - # host: ghe.example.net - # config: - # rawBaseUrl: https://ghe.example.net/raw - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN - - type: gitlab - host: gitlab.com - config: - token: - $secret: - env: GITLAB_PRIVATE_TOKEN - - type: bitbucket - host: bitbucket.org - config: - username: - $secret: - env: BITBUCKET_USERNAME - appPassword: - $secret: - env: BITBUCKET_APP_PASSWORD - - type: azure - host: dev.azure.com - config: - privateToken: - $secret: - env: AZURE_PRIVATE_TOKEN + # - host: ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + gitlab: + - host: gitlab.com + token: + $secret: + env: GITLAB_PRIVATE_TOKEN + bitbucket: + - host: bitbucket.org + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + azure: + - host: dev.azure.com + token: + $secret: + env: AZURE_PRIVATE_TOKEN catalog: rules: diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 33b5c68cae..b8e996fcd9 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -17,19 +17,18 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch'; -import { Logger } from 'winston'; import { NotFoundError } from '../errors'; import { UrlReader } from './types'; +import { ReaderFactory } from './UrlReaders'; /** * The configuration parameters for a single GitHub API provider. */ export type ProviderConfig = { /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. + * The host of the target that this matches on, e.g. "github.com" */ - target: string; + host: string; /** * The base URL of the API of this provider, e.g. "https://api.github.com", @@ -137,63 +136,47 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL { } } -export function readConfig(config: Config, logger: Logger): ProviderConfig[] { +export function readConfig(config: Config): ProviderConfig[] { const providers: ProviderConfig[] = []; - // TODO(freben): Deprecate the old config root entirely in a later release - if (config.has('catalog.processors.githubApi')) { - logger.warn( - 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', - ); - } - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. const providerConfigs = - config.getOptionalConfigArray('catalog.processors.github.providers') ?? - config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? - []; - const legacyToken = - config.getOptionalString('catalog.processors.github.privateToken') ?? - config.getOptionalString('catalog.processors.githubApi.privateToken'); + config.getOptionalConfigArray('integrations.github') ?? []; // First read all the explicit providers for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); + const host = providerConfig.getOptionalString('host') ?? 'github.com'; let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); const token = providerConfig.getOptionalString('token'); if (apiBaseUrl) { apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { + } else if (host === 'github.com') { apiBaseUrl = 'https://api.github.com'; } if (rawBaseUrl) { rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { + } else if (host === 'github.com') { rawBaseUrl = 'https://raw.githubusercontent.com'; } if (!apiBaseUrl && !rawBaseUrl) { throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + `GitHub integration for ${host} must configure an explicit apiBaseUrl and rawBaseUrl`, ); } - providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + providers.push({ host, apiBaseUrl, rawBaseUrl, token }); } // If no explicit github.com provider was added, put one in the list as // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { + if (!providers.some(p => p.host === 'github.com')) { providers.push({ - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', - token: legacyToken, }); } @@ -207,6 +190,14 @@ export function readConfig(config: Config, logger: Logger): ProviderConfig[] { export class GithubUrlReader implements UrlReader { private config: ProviderConfig; + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(provider => { + const reader = new GithubUrlReader(provider); + const predicate = (url: URL) => url.host === provider.host; + return { reader, predicate }; + }); + }; + constructor(config: ProviderConfig) { this.config = config; } diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 5aa6432cfc..6560775e32 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -14,16 +14,27 @@ * limitations under the License. */ +import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { GithubUrlReader } from './GithubUrlReader'; import { UrlReader } from './types'; -import { UrlReaderHostMux } from './UrlReaderHostMux'; +import { + UrlReaderPredicateMux, + UrlReaderPredicateTuple, +} from './UrlReaderPredicateMux'; export type ReaderFactoryOptions = { config: Config; logger: Logger; }; -export type ReaderFactory = (options: ReaderFactoryOptions) => UrlReader; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = ( + options: ReaderFactoryOptions, +) => UrlReaderPredicateTuple[]; /** * UrlReaders provide various utilities related to the UrlReader interface. @@ -33,22 +44,18 @@ export class UrlReaders { * Creates a new UrlReaders instance without any known types. */ static empty({ logger }: { logger: Logger }) { - return new UrlReaders(new Map(), logger); + return new UrlReaders([], logger); } /** - * Creates a new UrlReaders instance that includes factories for all types - * implemented in this package. + * Creates a new UrlReaders instance that includes all the default factories from this package */ static default({ logger }: { logger: Logger }) { - return new UrlReaders( - new Map([['github', GithubUrlReader.factory]]), - logger, - ); + return new UrlReaders([GithubUrlReader.factory], logger); } private constructor( - private readonly factories: Map, + private readonly factories: ReaderFactory[], private readonly logger: Logger, ) {} @@ -57,33 +64,29 @@ export class UrlReaders { * reader type needs to have a registered factory, or an error will be thrown. */ createWithConfig(config: Config): UrlReader { - const readerItems = config.getOptionalConfigArray('readers.byHost') ?? []; - const mux = new UrlReaderHostMux(); + const mux = new UrlReaderPredicateMux(); + const readers = []; - for (const readerItem of readerItems) { - const type = readerItem.getString('type'); - const host = readerItem.getString('host'); - const readerConfig = readerItem.getConfig('config'); + for (const factory of this.factories) { + const tuples = factory({ config, logger: this.logger }); - const factory = this.factories.get(type); - if (!factory) { - throw new Error(`Failed to create reader for unknown type '${type}'`); + for (const tuple of tuples) { + mux.register(tuple); + readers.push(tuple.reader); } - const reader = factory({ - config: readerConfig, - logger: this.logger, - }); - - mux.register(host, reader); } + this.logger.info( + `Registered the following UrlReaders: ${readers.join(', ')}`, + ); + return mux; } /** - * Register a UrlReader factory for a given type + * Register a UrlReader factory */ - register(type: string, factory: ReaderFactory) { - this.factories.set(type, factory); + addFactory(factory: ReaderFactory) { + this.factories.push(factory); } }