From bb553ed284e16c03ac11a04c33ee3d3f151b591b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 29 Sep 2020 12:56:31 +0200 Subject: [PATCH 01/33] backend-common: add UrlReader interface --- packages/backend-common/src/reading/index.ts | 17 +++++++++++++++ packages/backend-common/src/reading/types.ts | 22 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 packages/backend-common/src/reading/index.ts create mode 100644 packages/backend-common/src/reading/types.ts diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts new file mode 100644 index 0000000000..c632613934 --- /dev/null +++ b/packages/backend-common/src/reading/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { UrlReader } from './types'; diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts new file mode 100644 index 0000000000..9b29d1c45d --- /dev/null +++ b/packages/backend-common/src/reading/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A generic interface for fetching plain data from URLs. + */ +export type UrlReader = { + read(url: string): Promise; +}; From 1473523c8899df90c171bf2f4396eb648da57a30 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 29 Sep 2020 12:57:41 +0200 Subject: [PATCH 02/33] backend-common: add UrlReaderHostMux --- .../src/reading/UrlReaderHostMux.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 packages/backend-common/src/reading/UrlReaderHostMux.ts diff --git a/packages/backend-common/src/reading/UrlReaderHostMux.ts b/packages/backend-common/src/reading/UrlReaderHostMux.ts new file mode 100644 index 0000000000..446308c54c --- /dev/null +++ b/packages/backend-common/src/reading/UrlReaderHostMux.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from './types'; + +/** + * A UrlReader implementation that selects from a set of UrlReaders + * based on the host in the URL. + */ +export class UrlReaderHostMux implements UrlReader { + private readonly readers = new Map(); + + register(host: string, reader: UrlReader): void { + this.readers.set(host, reader); + } + + read(url: string): Promise { + const parsed = new URL(url); + + const reader = this.readers.get(parsed.host); + if (!reader) { + throw new Error(`No reader registered for host ${parsed.host}`); + } + + return reader.read(url); + } +} From 8a3e63e5900f288fa4c6e4cdfa5ec60d16a008f4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 29 Sep 2020 18:53:48 +0200 Subject: [PATCH 03/33] catalog-backend: move GithubReaderProcessor to backend-common readers --- .../backend-common/src/reading/GithubUrlReader.test.ts | 0 .../backend-common/src/reading/GithubUrlReader.ts | 0 plugins/catalog-backend/src/ingestion/LocationReaders.ts | 3 --- 3 files changed, 3 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts => packages/backend-common/src/reading/GithubUrlReader.test.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts => packages/backend-common/src/reading/GithubUrlReader.ts (100%) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts rename to packages/backend-common/src/reading/GithubUrlReader.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/packages/backend-common/src/reading/GithubUrlReader.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts rename to packages/backend-common/src/reading/GithubUrlReader.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 00ee95f3a5..0ae6135cae 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,7 +32,6 @@ import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProc import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; -import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; @@ -76,14 +75,12 @@ export class LocationReaders implements LocationReader { entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { - logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - GithubReaderProcessor.fromConfig(config, logger), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), From b461b367d7cf9c99f7b39fd89309f7bb8deec6b0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 29 Sep 2020 18:46:25 +0200 Subject: [PATCH 04/33] backend-common: port GithubUrlReader --- packages/backend-common/package.json | 3 + .../src/reading/GithubUrlReader.test.ts | 37 +++----- .../src/reading/GithubUrlReader.ts | 88 +++++++------------ 3 files changed, 43 insertions(+), 85 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 72f047761d..b2550c6551 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -39,11 +39,13 @@ "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", + "git-url-parse": "^11.2.0", "helmet": "^4.0.0", "knex": "^0.21.1", "lodash": "^4.17.15", "logform": "^2.1.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.0", "prom-client": "^12.0.0", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", @@ -62,6 +64,7 @@ "@types/compression": "^1.7.0", "@types/http-errors": "^1.6.3", "@types/morgan": "^1.9.0", + "@types/node-fetch": "^2.5.7", "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", "@types/webpack-env": "^1.15.2", diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 8eb46db967..513d66c09b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -14,20 +14,19 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; -import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; import { getApiRequestOptions, getApiUrl, getRawRequestOptions, getRawUrl, - GithubReaderProcessor, + GithubUrlReader, ProviderConfig, readConfig, -} from './GithubReaderProcessor'; +} from './GithubUrlReader'; -describe('GithubReaderProcessor', () => { +describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { it('sets the correct API version', () => { const config: ProviderConfig = { target: '', apiBaseUrl: '' }; @@ -238,31 +237,15 @@ describe('GithubReaderProcessor', () => { }); describe('implementation', () => { - it('rejects unknown types', async () => { - const processor = new GithubReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'not-github', - target: 'https://github.com', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).resolves.toBeFalsy(); - }); - it('rejects unknown targets', async () => { - const processor = new GithubReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'github', - target: 'https://not.github.com/apa', - }; + const processor = new GithubUrlReader({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }); await expect( - processor.readLocation(location, false, () => {}), + processor.read('https://not.github.com/apa'), ).rejects.toThrow( - /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + 'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path', ); }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index cc50e53240..33b5c68cae 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; -import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch'; import { Logger } from 'winston'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; +import { NotFoundError } from '../errors'; +import { UrlReader } from './types'; /** * The configuration parameters for a single GitHub API provider. @@ -101,7 +100,7 @@ export function getApiUrl(target: string, provider: ProviderConfig): URL { !ref || (filepathtype !== 'blob' && filepathtype !== 'raw') ) { - throw new Error('Wrong URL or invalid file path'); + throw new Error('Invalid GitHub URL or file path'); } const pathWithoutSlash = filepath.replace(/^\//, ''); @@ -126,7 +125,7 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL { !ref || (filepathtype !== 'blob' && filepathtype !== 'raw') ) { - throw new Error('Wrong URL or invalid file path'); + throw new Error('Invalid GitHub URL or file path'); } const pathWithoutSlash = filepath.replace(/^\//, ''); @@ -205,65 +204,38 @@ export function readConfig(config: Config, logger: Logger): ProviderConfig[] { * A processor that adds the ability to read files from GitHub v3 APIs, such as * the one exposed by GitHub itself. */ -export class GithubReaderProcessor implements LocationProcessor { - private providers: ProviderConfig[]; +export class GithubUrlReader implements UrlReader { + private config: ProviderConfig; - static fromConfig(config: Config, logger: Logger) { - return new GithubReaderProcessor(readConfig(config, logger)); + constructor(config: ProviderConfig) { + this.config = config; } - constructor(providers: ProviderConfig[]) { - this.providers = providers; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - // The github/api type is for backward compatibility - if (location.type !== 'github' && location.type !== 'github/api') { - return false; - } - - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, - ); - } + async read(url: string): Promise { + const useApi = + this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl); + const ghUrl = useApi + ? getApiUrl(url, this.config) + : getRawUrl(url, this.config); + const options = useApi + ? getApiRequestOptions(this.config) + : getRawRequestOptions(this.config); + let response: Response; try { - const useApi = - provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); - const url = useApi - ? getApiUrl(location.target, provider) - : getRawUrl(location.target, provider); - const options = useApi - ? getApiRequestOptions(provider) - : getRawRequestOptions(provider); - const response = await fetch(url.toString(), options); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } + response = await fetch(ghUrl.toString(), options); } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); + throw new Error(`Unable to read ${url}, ${e}`); } - return true; + if (response.ok) { + return response.buffer(); + } + + const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); } } From 2b7c1b70f9dcf5708b3f9f063bc6f341bc90012a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Sep 2020 10:42:14 +0200 Subject: [PATCH 05/33] app-config: move catalog processor configs to new readers key --- app-config.yaml | 82 +++++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 88df40813e..d4464332fa 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -67,28 +67,59 @@ kubernetes: clusterLocatorMethod: 'configMultiTenant' clusters: [] +# This sections configures the URL readers used by various parts of +# Backstage to read data from remote URLs +readers: + byHost: + - type: github + host: github.com + config: + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + ### Example for how to add your GitHub Enterprise instance using the API: + # - type: github + # host: ghe.example.net + # config: + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - type: github + # host: ghe.example.net + # config: + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + - type: gitlab + host: gitlab.com + config: + token: + $secret: + env: GITLAB_PRIVATE_TOKEN + - type: bitbucket + host: bitbucket.org + config: + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + - type: azure + host: dev.azure.com + config: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN + catalog: rules: - allow: [Component, API, Group, User, Template, Location] + processors: - github: - providers: - - target: https://github.com - token: - $secret: - env: GITHUB_PRIVATE_TOKEN - #### Example for how to add your GitHub Enterprise instance using the API: - # - target: https://ghe.example.net - # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN - #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): - # - target: https://ghe.example.net - # rawBaseUrl: https://ghe.example.net/raw - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN githubOrg: providers: - target: https://github.com @@ -101,21 +132,6 @@ catalog: # token: # $secret: # env: GHE_PRIVATE_TOKEN - bitbucketApi: - username: - $secret: - env: BITBUCKET_USERNAME - appPassword: - $secret: - env: BITBUCKET_APP_PASSWORD - gitlabApi: - privateToken: - $secret: - env: GITLAB_PRIVATE_TOKEN - azureApi: - privateToken: - $secret: - env: AZURE_PRIVATE_TOKEN locations: # Backstage example components From 44c8816db51ca1d10f153fa3491cd962999aeaec Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Sep 2020 11:40:22 +0200 Subject: [PATCH 06/33] backend-common: add UrlReaders for constructing a reader from config --- packages/backend-common/src/index.ts | 1 + .../backend-common/src/reading/UrlReaders.ts | 89 +++++++++++++++++++ packages/backend-common/src/reading/index.ts | 1 + 3 files changed, 91 insertions(+) create mode 100644 packages/backend-common/src/reading/UrlReaders.ts diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 7074040655..e968edef69 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -20,6 +20,7 @@ export * from './discovery'; export * from './errors'; export * from './logging'; export * from './middleware'; +export * from './reading'; export * from './service'; export * from './paths'; export * from './hot'; diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts new file mode 100644 index 0000000000..5aa6432cfc --- /dev/null +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { GithubUrlReader } from './GithubUrlReader'; +import { UrlReader } from './types'; +import { UrlReaderHostMux } from './UrlReaderHostMux'; + +export type ReaderFactoryOptions = { + config: Config; + logger: Logger; +}; +export type ReaderFactory = (options: ReaderFactoryOptions) => UrlReader; + +/** + * UrlReaders provide various utilities related to the UrlReader interface. + */ +export class UrlReaders { + /** + * Creates a new UrlReaders instance without any known types. + */ + static empty({ logger }: { logger: Logger }) { + return new UrlReaders(new Map(), logger); + } + + /** + * Creates a new UrlReaders instance that includes factories for all types + * implemented in this package. + */ + static default({ logger }: { logger: Logger }) { + return new UrlReaders( + new Map([['github', GithubUrlReader.factory]]), + logger, + ); + } + + private constructor( + private readonly factories: Map, + private readonly logger: Logger, + ) {} + + /** + * Constructs a new UrlReader using the provided configuration. Any encountered + * reader type needs to have a registered factory, or an error will be thrown. + */ + createWithConfig(config: Config): UrlReader { + const readerItems = config.getOptionalConfigArray('readers.byHost') ?? []; + const mux = new UrlReaderHostMux(); + + for (const readerItem of readerItems) { + const type = readerItem.getString('type'); + const host = readerItem.getString('host'); + const readerConfig = readerItem.getConfig('config'); + + const factory = this.factories.get(type); + if (!factory) { + throw new Error(`Failed to create reader for unknown type '${type}'`); + } + const reader = factory({ + config: readerConfig, + logger: this.logger, + }); + + mux.register(host, reader); + } + + return mux; + } + + /** + * Register a UrlReader factory for a given type + */ + register(type: string, factory: ReaderFactory) { + this.factories.set(type, factory); + } +} diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index c632613934..ad493c34fd 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -15,3 +15,4 @@ */ export type { UrlReader } from './types'; +export { UrlReaders } from './UrlReaders'; From f2ad1b6f276f642c30f7fd2553d4f6b2d03871ee Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Sep 2020 19:41:48 +0200 Subject: [PATCH 07/33] catalog-backend: switch UrlReaderProcessor to use UrlReader --- .../src/ingestion/LocationReaders.ts | 11 +++++--- .../processors/UrlReaderProcessor.ts | 28 ++++++++----------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0ae6135cae..a217e07f1d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; import { Entity, EntityPolicies, @@ -56,6 +56,7 @@ import { LocationReader, ReadLocationResult } from './types'; const MAX_DEPTH = 10; type Options = { + reader?: UrlReader; logger?: Logger; config?: Config; processors?: LocationProcessor[]; @@ -71,6 +72,7 @@ export class LocationReaders implements LocationReader { static defaultProcessors(options: { logger: Logger; + reader?: UrlReader; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { @@ -86,7 +88,7 @@ export class LocationReaders implements LocationReader { new BitbucketApiReaderProcessor(config), new AzureApiReaderProcessor(config), GithubOrgReaderProcessor.fromConfig(config), - new UrlReaderProcessor(), + options.reader ? new UrlReaderProcessor(options.reader) : [], new YamlProcessor(), PlaceholderProcessor.default(), new CodeOwnersProcessor(), @@ -94,13 +96,14 @@ export class LocationReaders implements LocationReader { new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), new AnnotateLocationEntityProcessor(), - ]; + ].flat(); } constructor({ logger = getVoidLogger(), config, - processors = LocationReaders.defaultProcessors({ logger, config }), + reader, + processors = LocationReaders.defaultProcessors({ logger, reader, config }), }: Options) { this.logger = logger; this.processors = processors; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 0c879ea70c..94a7b01336 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; export class UrlReaderProcessor implements LocationProcessor { + constructor(private readonly reader: UrlReader) {} + async readLocation( location: LocationSpec, optional: boolean, @@ -30,24 +32,18 @@ export class UrlReaderProcessor implements LocationProcessor { } try { - const response = await fetch(location.target); + const data = await this.reader.read(location.target); + emit(result.data(location, data)); + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); } + } else { + emit(result.generalError(location, message)); } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); } return true; From 0fbe965b436021864b17265c6d373c3355f354a2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 10:35:56 +0200 Subject: [PATCH 08/33] backend-common: add UrlReaderPredicateMux --- .../src/reading/UrlReaderPredicateMux.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/backend-common/src/reading/UrlReaderPredicateMux.ts diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts new file mode 100644 index 0000000000..a02c7df86c --- /dev/null +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from './types'; + +export type UrlReaderPredicate = (url: URL) => boolean; + +export type UrlReaderPredicateTuple = { + predicate: UrlReaderPredicate; + reader: UrlReader; +}; + +/** + * A UrlReader implementation that selects from a set of UrlReaders + * based on a predicate tied to each reader. + */ +export class UrlReaderPredicateMux implements UrlReader { + private readonly readers: UrlReaderPredicateTuple[] = []; + + register(tuple: UrlReaderPredicateTuple): void { + this.readers.push(tuple); + } + + read(url: string): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + return reader.read(url); + } + } + + throw new Error(`No reader found that could handle '${url}'`); + } +} From 41d55e8d76064fab3fd4e8c0ff7ebdc53cb58009 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 13:36:07 +0200 Subject: [PATCH 09/33] backend-common: refactor UrlReaders and factories to be centered around predicates --- app-config.yaml | 79 ++++++++----------- .../src/reading/GithubUrlReader.ts | 49 +++++------- .../backend-common/src/reading/UrlReaders.ts | 59 +++++++------- 3 files changed, 85 insertions(+), 102 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index d4464332fa..d21e858e8a 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -67,53 +67,42 @@ kubernetes: clusterLocatorMethod: 'configMultiTenant' clusters: [] -# This sections configures the URL readers used by various parts of -# Backstage to read data from remote URLs -readers: - byHost: - - type: github - host: github.com - config: - token: - $secret: - env: GITHUB_PRIVATE_TOKEN +integrations: + github: + - host: github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN ### Example for how to add your GitHub Enterprise instance using the API: - # - type: github - # host: ghe.example.net - # config: - # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN + # - host: ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): - # - type: github - # host: ghe.example.net - # config: - # rawBaseUrl: https://ghe.example.net/raw - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN - - type: gitlab - host: gitlab.com - config: - token: - $secret: - env: GITLAB_PRIVATE_TOKEN - - type: bitbucket - host: bitbucket.org - config: - username: - $secret: - env: BITBUCKET_USERNAME - appPassword: - $secret: - env: BITBUCKET_APP_PASSWORD - - type: azure - host: dev.azure.com - config: - privateToken: - $secret: - env: AZURE_PRIVATE_TOKEN + # - host: ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + gitlab: + - host: gitlab.com + token: + $secret: + env: GITLAB_PRIVATE_TOKEN + bitbucket: + - host: bitbucket.org + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + azure: + - host: dev.azure.com + token: + $secret: + env: AZURE_PRIVATE_TOKEN catalog: rules: diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 33b5c68cae..b8e996fcd9 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -17,19 +17,18 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch'; -import { Logger } from 'winston'; import { NotFoundError } from '../errors'; import { UrlReader } from './types'; +import { ReaderFactory } from './UrlReaders'; /** * The configuration parameters for a single GitHub API provider. */ export type ProviderConfig = { /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. + * The host of the target that this matches on, e.g. "github.com" */ - target: string; + host: string; /** * The base URL of the API of this provider, e.g. "https://api.github.com", @@ -137,63 +136,47 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL { } } -export function readConfig(config: Config, logger: Logger): ProviderConfig[] { +export function readConfig(config: Config): ProviderConfig[] { const providers: ProviderConfig[] = []; - // TODO(freben): Deprecate the old config root entirely in a later release - if (config.has('catalog.processors.githubApi')) { - logger.warn( - 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', - ); - } - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. const providerConfigs = - config.getOptionalConfigArray('catalog.processors.github.providers') ?? - config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? - []; - const legacyToken = - config.getOptionalString('catalog.processors.github.privateToken') ?? - config.getOptionalString('catalog.processors.githubApi.privateToken'); + config.getOptionalConfigArray('integrations.github') ?? []; // First read all the explicit providers for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); + const host = providerConfig.getOptionalString('host') ?? 'github.com'; let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); const token = providerConfig.getOptionalString('token'); if (apiBaseUrl) { apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { + } else if (host === 'github.com') { apiBaseUrl = 'https://api.github.com'; } if (rawBaseUrl) { rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { + } else if (host === 'github.com') { rawBaseUrl = 'https://raw.githubusercontent.com'; } if (!apiBaseUrl && !rawBaseUrl) { throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + `GitHub integration for ${host} must configure an explicit apiBaseUrl and rawBaseUrl`, ); } - providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + providers.push({ host, apiBaseUrl, rawBaseUrl, token }); } // If no explicit github.com provider was added, put one in the list as // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { + if (!providers.some(p => p.host === 'github.com')) { providers.push({ - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', - token: legacyToken, }); } @@ -207,6 +190,14 @@ export function readConfig(config: Config, logger: Logger): ProviderConfig[] { export class GithubUrlReader implements UrlReader { private config: ProviderConfig; + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(provider => { + const reader = new GithubUrlReader(provider); + const predicate = (url: URL) => url.host === provider.host; + return { reader, predicate }; + }); + }; + constructor(config: ProviderConfig) { this.config = config; } diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 5aa6432cfc..6560775e32 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -14,16 +14,27 @@ * limitations under the License. */ +import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { GithubUrlReader } from './GithubUrlReader'; import { UrlReader } from './types'; -import { UrlReaderHostMux } from './UrlReaderHostMux'; +import { + UrlReaderPredicateMux, + UrlReaderPredicateTuple, +} from './UrlReaderPredicateMux'; export type ReaderFactoryOptions = { config: Config; logger: Logger; }; -export type ReaderFactory = (options: ReaderFactoryOptions) => UrlReader; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = ( + options: ReaderFactoryOptions, +) => UrlReaderPredicateTuple[]; /** * UrlReaders provide various utilities related to the UrlReader interface. @@ -33,22 +44,18 @@ export class UrlReaders { * Creates a new UrlReaders instance without any known types. */ static empty({ logger }: { logger: Logger }) { - return new UrlReaders(new Map(), logger); + return new UrlReaders([], logger); } /** - * Creates a new UrlReaders instance that includes factories for all types - * implemented in this package. + * Creates a new UrlReaders instance that includes all the default factories from this package */ static default({ logger }: { logger: Logger }) { - return new UrlReaders( - new Map([['github', GithubUrlReader.factory]]), - logger, - ); + return new UrlReaders([GithubUrlReader.factory], logger); } private constructor( - private readonly factories: Map, + private readonly factories: ReaderFactory[], private readonly logger: Logger, ) {} @@ -57,33 +64,29 @@ export class UrlReaders { * reader type needs to have a registered factory, or an error will be thrown. */ createWithConfig(config: Config): UrlReader { - const readerItems = config.getOptionalConfigArray('readers.byHost') ?? []; - const mux = new UrlReaderHostMux(); + const mux = new UrlReaderPredicateMux(); + const readers = []; - for (const readerItem of readerItems) { - const type = readerItem.getString('type'); - const host = readerItem.getString('host'); - const readerConfig = readerItem.getConfig('config'); + for (const factory of this.factories) { + const tuples = factory({ config, logger: this.logger }); - const factory = this.factories.get(type); - if (!factory) { - throw new Error(`Failed to create reader for unknown type '${type}'`); + for (const tuple of tuples) { + mux.register(tuple); + readers.push(tuple.reader); } - const reader = factory({ - config: readerConfig, - logger: this.logger, - }); - - mux.register(host, reader); } + this.logger.info( + `Registered the following UrlReaders: ${readers.join(', ')}`, + ); + return mux; } /** - * Register a UrlReader factory for a given type + * Register a UrlReader factory */ - register(type: string, factory: ReaderFactory) { - this.factories.set(type, factory); + addFactory(factory: ReaderFactory) { + this.factories.push(factory); } } From 683181837becaff6df678bda977856cb1e24c5e4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 13:46:35 +0200 Subject: [PATCH 10/33] catalog-backend: move over remaining to-be UrlReaders --- .../backend-common/src/reading/AzureUrlReader.test.ts | 0 .../backend-common/src/reading/AzureUrlReader.ts | 0 .../backend-common/src/reading/BitbucketUrlReader.test.ts | 0 .../backend-common/src/reading/BitbucketUrlReader.ts | 0 .../backend-common/src/reading}/GitlabReaderProcessor.ts | 0 .../backend-common/src/reading/GitlabUrlReader.test.ts | 0 .../backend-common/src/reading/GitlabUrlReader.ts | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts => packages/backend-common/src/reading/AzureUrlReader.test.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts => packages/backend-common/src/reading/AzureUrlReader.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts => packages/backend-common/src/reading/BitbucketUrlReader.test.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts => packages/backend-common/src/reading/BitbucketUrlReader.ts (100%) rename {plugins/catalog-backend/src/ingestion/processors => packages/backend-common/src/reading}/GitlabReaderProcessor.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts => packages/backend-common/src/reading/GitlabUrlReader.test.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts => packages/backend-common/src/reading/GitlabUrlReader.ts (100%) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts rename to packages/backend-common/src/reading/AzureUrlReader.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/packages/backend-common/src/reading/AzureUrlReader.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts rename to packages/backend-common/src/reading/AzureUrlReader.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts rename to packages/backend-common/src/reading/BitbucketUrlReader.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts rename to packages/backend-common/src/reading/BitbucketUrlReader.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/packages/backend-common/src/reading/GitlabReaderProcessor.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts rename to packages/backend-common/src/reading/GitlabReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts rename to packages/backend-common/src/reading/GitlabUrlReader.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts rename to packages/backend-common/src/reading/GitlabUrlReader.ts From 4cdac4624e414b1bec67bdfea8caf70af6b32a26 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 14:50:59 +0200 Subject: [PATCH 11/33] backend-common: port BitbucketUrlReader --- .../src/reading/BitbucketUrlReader.test.ts | 35 +++--- .../src/reading/BitbucketUrlReader.ts | 116 +++++++++++------- .../backend-common/src/reading/UrlReaders.ts | 6 +- packages/backend-common/src/reading/index.ts | 2 + 4 files changed, 98 insertions(+), 61 deletions(-) diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 953f0703be..43f797edd4 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,10 +14,12 @@ * limitations under the License. */ -import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; +import { BitbucketUrlReader, readConfig } from './BitbucketUrlReader'; import { ConfigReader } from '@backstage/config'; -describe('BitbucketApiReaderProcessor', () => { +const host = 'bitbucket.org'; + +describe('BitbucketUrlReader', () => { const createConfig = ( username: string | undefined, appPassword: string | undefined, @@ -26,22 +28,21 @@ describe('BitbucketApiReaderProcessor', () => { { context: '', data: { - catalog: { - processors: { - bitbucketApi: { + integrations: { + bitbucket: [ + { + host, username: username, appPassword: appPassword, }, - }, + ], }, }, }, ]); it('should build raw api', () => { - const processor = new BitbucketApiReaderProcessor( - createConfig(undefined, undefined), - ); + const processor = new BitbucketUrlReader({ host }); const tests = [ { @@ -90,7 +91,7 @@ describe('BitbucketApiReaderProcessor', () => { headers: {}, }, err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", + "Invalid type in config for key 'integrations.bitbucket[0].username' in '', got empty-string, wanted string", }, { username: 'only-user-provided', @@ -99,7 +100,7 @@ describe('BitbucketApiReaderProcessor', () => { headers: {}, }, err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string", + "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in '', got empty-string, wanted string", }, { username: '', @@ -108,7 +109,7 @@ describe('BitbucketApiReaderProcessor', () => { headers: {}, }, err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", + "Invalid type in config for key 'integrations.bitbucket[0].username' in '', got empty-string, wanted string", }, { username: 'some-user', @@ -132,6 +133,8 @@ describe('BitbucketApiReaderProcessor', () => { expect: { headers: {}, }, + err: + "Missing required config value at 'integrations.bitbucket[0].appPassword'", }, { username: undefined, @@ -146,13 +149,13 @@ describe('BitbucketApiReaderProcessor', () => { if (test.err) { expect( () => - new BitbucketApiReaderProcessor( - createConfig(test.username, test.password), + new BitbucketUrlReader( + readConfig(createConfig(test.username, test.password))[0], ), ).toThrowError(test.err); } else { - const processor = new BitbucketApiReaderProcessor( - createConfig(test.username, test.password), + const processor = new BitbucketUrlReader( + readConfig(createConfig(test.username, test.password))[0], ); expect(processor.getRequestOptions()).toEqual(test.expect); } diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index c46fbd1737..62739ded1c 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -14,31 +14,72 @@ * limitations under the License. */ -import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; +import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; +import { ReaderFactory } from './UrlReaders'; +import { UrlReader } from './types'; +import { NotFoundError } from '../errors'; -export class BitbucketApiReaderProcessor implements LocationProcessor { - private username: string; - private password: string; +type Options = { + // TODO: added here for future support, but we only allow bitbucket.org for now + host: string; + auth?: { + username: string; + appPassword: string; + }; +}; - constructor(config: Config) { - this.username = - config.getOptionalString('catalog.processors.bitbucketApi.username') ?? - ''; - this.password = - config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? - ''; +export function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.bitbucket') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org'; + + let auth; + if (providerConfig.has('username')) { + const username = providerConfig.getString('username'); + const appPassword = providerConfig.getString('appPassword'); + auth = { username, appPassword }; + } + + optionsArr.push({ host, auth }); + } + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public bitbucket repos. + if (!optionsArr.some(p => p.host === 'bitbucket.org')) { + optionsArr.push({ host: 'bitbucket.org' }); + } + + return optionsArr; +} + +export class BitbucketUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new BitbucketUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) { + if (options.host !== 'bitbucket.org') { + throw Error( + `Bitbucket integration currently only supports 'bitbucket.org', tried to use host '${options.host}'`, + ); + } } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; - if (this.username !== '' && this.password !== '') { + if (this.options.auth) { headers.Authorization = `Basic ${Buffer.from( - `${this.username}:${this.password}`, + `${this.options.auth.username}:${this.options.auth.appPassword}`, 'utf8', ).toString('base64')}`; } @@ -50,38 +91,25 @@ export class BitbucketApiReaderProcessor implements LocationProcessor { return requestOptions; } - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'bitbucket/api') { - return false; - } + async read(url: string): Promise { + const builtUrl = this.buildRawUrl(url); + let response: Response; try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } + response = await fetch(builtUrl.toString(), this.getRequestOptions()); } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); + throw new Error(`Unable to read ${url}, ${e}`); } - return true; + + if (response.ok) { + return response.buffer(); + } + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); } // Converts diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 6560775e32..1934138d3c 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -22,6 +22,7 @@ import { UrlReaderPredicateMux, UrlReaderPredicateTuple, } from './UrlReaderPredicateMux'; +import { BitbucketUrlReader } from './BitbucketUrlReader'; export type ReaderFactoryOptions = { config: Config; @@ -51,7 +52,10 @@ export class UrlReaders { * Creates a new UrlReaders instance that includes all the default factories from this package */ static default({ logger }: { logger: Logger }) { - return new UrlReaders([GithubUrlReader.factory], logger); + return new UrlReaders( + [GithubUrlReader.factory, BitbucketUrlReader.factory], + logger, + ); } private constructor( diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index ad493c34fd..286bdd6786 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -16,3 +16,5 @@ export type { UrlReader } from './types'; export { UrlReaders } from './UrlReaders'; +export { GithubUrlReader } from './GithubUrlReader'; +export { BitbucketUrlReader } from './BitbucketUrlReader'; From ce32eb3c697eab3e576d00c9d9c9545333fb18fc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 15:02:06 +0200 Subject: [PATCH 12/33] backend-common: port AzureUrlReader --- .../src/reading/AzureUrlReader.test.ts | 29 +++-- .../src/reading/AzureUrlReader.ts | 105 +++++++++++------- .../backend-common/src/reading/UrlReaders.ts | 9 +- packages/backend-common/src/reading/index.ts | 3 +- 4 files changed, 91 insertions(+), 55 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 9b234e3e75..50db27b314 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -14,28 +14,33 @@ * limitations under the License. */ -import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; +import { AzureUrlReader, readConfig } from './AzureUrlReader'; import { ConfigReader } from '@backstage/config'; -describe('AzureApiReaderProcessor', () => { +const host = 'dev.azure.com'; + +describe('AzureUrlReader', () => { const createConfig = (token: string | undefined) => ConfigReader.fromConfigs([ { context: '', data: { - catalog: { - processors: { - azureApi: { - privateToken: token, + integrations: { + azure: [ + { + host, + token, }, - }, + ], }, }, }, ]); it('should build raw api', () => { - const processor = new AzureApiReaderProcessor(createConfig(undefined)); + const processor = new AzureUrlReader( + readConfig(createConfig(undefined))[0], + ); const tests = [ { target: @@ -98,7 +103,7 @@ describe('AzureApiReaderProcessor', () => { headers: {}, }, err: - "Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string", + "Invalid type in config for key 'integrations.azure[0].token' in '', got empty-string, wanted string", }, { token: undefined, @@ -111,10 +116,12 @@ describe('AzureApiReaderProcessor', () => { for (const test of tests) { if (test.err) { expect( - () => new AzureApiReaderProcessor(createConfig(test.token)), + () => new AzureUrlReader(readConfig(createConfig(test.token))[0]), ).toThrowError(test.err); } else { - const processor = new AzureApiReaderProcessor(createConfig(test.token)); + const processor = new AzureUrlReader( + readConfig(createConfig(test.token))[0], + ); expect(processor.getRequestOptions()).toEqual(test.expect); } } diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 70a84d72df..202695eaed 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -14,27 +14,63 @@ * limitations under the License. */ -import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; +import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; +import { NotFoundError } from '../errors'; +import { UrlReader } from './types'; +import { ReaderFactory } from './UrlReaders'; -export class AzureApiReaderProcessor implements LocationProcessor { - private privateToken: string; +type Options = { + // TODO: added here for future support, but we only allow dev.azure.com for now + host: string; + token?: string; +}; - constructor(config: Config) { - this.privateToken = - config.getOptionalString('catalog.processors.azureApi.privateToken') ?? - ''; +export function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.azure') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'dev.azure.com'; + const token = providerConfig.getOptionalString('token'); + + optionsArr.push({ host, token }); + } + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public azure repos. + if (!optionsArr.some(p => p.host === 'dev.azure.com')) { + optionsArr.push({ host: 'dev.azure.com' }); + } + + return optionsArr; +} + +export class AzureUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new AzureUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) { + if (options.host !== 'dev.azure.com') { + throw Error( + `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, + ); + } } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; - if (this.privateToken !== '') { + if (this.options.token) { headers.Authorization = `Basic ${Buffer.from( - `:${this.privateToken}`, + `:${this.options.token}`, 'utf8', ).toString('base64')}`; } @@ -46,39 +82,26 @@ export class AzureApiReaderProcessor implements LocationProcessor { return requestOptions; } - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'azure/api') { - return false; - } + async read(url: string): Promise { + const builtUrl = this.buildRawUrl(url); + let response: Response; try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html - if (response.ok && response.status !== 203) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } + response = await fetch(builtUrl.toString(), this.getRequestOptions()); } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); + throw new Error(`Unable to read ${url}, ${e}`); } - return true; + + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { + return response.buffer(); + } + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); } // Converts diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 1934138d3c..f88c3bfacc 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -16,13 +16,14 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { GithubUrlReader } from './GithubUrlReader'; import { UrlReader } from './types'; import { UrlReaderPredicateMux, UrlReaderPredicateTuple, } from './UrlReaderPredicateMux'; +import { AzureUrlReader } from './AzureUrlReader'; import { BitbucketUrlReader } from './BitbucketUrlReader'; +import { GithubUrlReader } from './GithubUrlReader'; export type ReaderFactoryOptions = { config: Config; @@ -53,7 +54,11 @@ export class UrlReaders { */ static default({ logger }: { logger: Logger }) { return new UrlReaders( - [GithubUrlReader.factory, BitbucketUrlReader.factory], + [ + GithubUrlReader.factory, + BitbucketUrlReader.factory, + AzureUrlReader.factory, + ], logger, ); } diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 286bdd6786..d1dc1e7952 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -16,5 +16,6 @@ export type { UrlReader } from './types'; export { UrlReaders } from './UrlReaders'; -export { GithubUrlReader } from './GithubUrlReader'; +export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; +export { GithubUrlReader } from './GithubUrlReader'; From f50a233a4fcb89a0754ca7d17c37a338cd1722d2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 15:37:29 +0200 Subject: [PATCH 13/33] backend-common: port GitlabUrlReader --- .../src/reading/GitlabUrlReader.test.ts | 27 +++-- .../src/reading/GitlabUrlReader.ts | 108 ++++++++++-------- .../backend-common/src/reading/UrlReaders.ts | 6 +- packages/backend-common/src/reading/index.ts | 1 + 4 files changed, 81 insertions(+), 61 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 7e55fece1b..17023d9120 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -14,28 +14,31 @@ * limitations under the License. */ -import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; +import { GitlabUrlReader, readConfig } from './GitlabUrlReader'; import { ConfigReader } from '@backstage/config'; -describe('GitlabApiReaderProcessor', () => { +describe('GitlabUrlReader', () => { const createConfig = (token: string | undefined) => ConfigReader.fromConfigs([ { context: '', data: { - catalog: { - processors: { - gitlabApi: { - privateToken: token, + integrations: { + gitlab: [ + { + host: 'gitlab.com', + token: token, }, - }, + ], }, }, }, ]); it('should build raw api', () => { - const processor = new GitlabApiReaderProcessor(createConfig(undefined)); + const processor = new GitlabUrlReader( + readConfig(createConfig(undefined))[0], + ); const tests = [ { @@ -90,7 +93,7 @@ describe('GitlabApiReaderProcessor', () => { { token: '', err: - "Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string", + "Invalid type in config for key 'integrations.gitlab[0].token' in '', got empty-string, wanted string", expect: { headers: { 'PRIVATE-TOKEN': '', @@ -110,11 +113,11 @@ describe('GitlabApiReaderProcessor', () => { for (const test of tests) { if (test.err) { expect( - () => new GitlabApiReaderProcessor(createConfig(test.token)), + () => new GitlabUrlReader(readConfig(createConfig(test.token))[0]), ).toThrowError(test.err); } else { - const processor = new GitlabApiReaderProcessor( - createConfig(test.token), + const processor = new GitlabUrlReader( + readConfig(createConfig(test.token))[0], ); expect(processor.getRequestOptions()).toEqual(test.expect); } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index a51c306777..56ae91bb4b 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -14,65 +14,79 @@ * limitations under the License. */ -import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; +import fetch, { RequestInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; +import { NotFoundError } from '../errors'; +import { UrlReader } from './types'; +import { ReaderFactory } from './UrlReaders'; -export class GitlabApiReaderProcessor implements LocationProcessor { - private privateToken: string; +type Options = { + // TODO: added here for future support, but we only allow bitbucket.org for now + host: string; + token?: string; +}; - constructor(config: Config) { - this.privateToken = - config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? - ''; +export function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.gitlab') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'gitlab.com'; + const token = providerConfig.getOptionalString('token'); + + optionsArr.push({ host, token }); } + // As a convenience we always make sure there's at least an unauthenticated + // reader for public gitlab repos. + if (!optionsArr.some(p => p.host === 'gitlab.com')) { + optionsArr.push({ host: 'gitlab.com' }); + } + + return optionsArr; +} + +export class GitlabUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new GitlabUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) {} + getRequestOptions(): RequestInit { - const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; - if (this.privateToken !== '') { - headers['PRIVATE-TOKEN'] = this.privateToken; - } - - const requestOptions: RequestInit = { - headers, + return { + headers: { + ['PRIVATE-TOKEN']: this.options.token ?? '', + }, }; - - return requestOptions; } - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'gitlab/api') { - return false; + async read(url: string): Promise { + const projectID = await this.getProjectID(url); + const builtUrl = this.buildRawUrl(url, projectID); + + let response: Response; + try { + response = await fetch(builtUrl.toString(), this.getRequestOptions()); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); } - try { - const projectID = await this.getProjectID(location.target); - const url = this.buildRawUrl(location.target, projectID); - const response = await fetch(url.toString(), this.getRequestOptions()); - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); + if (response.ok) { + return response.buffer(); } - return true; + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); } // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index f88c3bfacc..2535c953e9 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -24,6 +24,7 @@ import { import { AzureUrlReader } from './AzureUrlReader'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { GithubUrlReader } from './GithubUrlReader'; +import { GitlabUrlReader } from './GitlabUrlReader'; export type ReaderFactoryOptions = { config: Config; @@ -55,9 +56,10 @@ export class UrlReaders { static default({ logger }: { logger: Logger }) { return new UrlReaders( [ - GithubUrlReader.factory, - BitbucketUrlReader.factory, AzureUrlReader.factory, + BitbucketUrlReader.factory, + GithubUrlReader.factory, + GitlabUrlReader.factory, ], logger, ); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index d1dc1e7952..8ecc08ebca 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,3 +19,4 @@ export { UrlReaders } from './UrlReaders'; export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; +export { GitlabUrlReader } from './GitlabUrlReader'; From 1e97419d305418c224b916c8a9d8c1cd00d9b9cc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 15:50:49 +0200 Subject: [PATCH 14/33] backend-common: merge GitlabReaderProcessor functionality into GitlabUrlReader --- .../src/reading/GitlabReaderProcessor.ts | 89 ------------------- .../src/reading/GitlabUrlReader.test.ts | 30 ++++++- .../src/reading/GitlabUrlReader.ts | 46 +++++++++- 3 files changed, 71 insertions(+), 94 deletions(-) delete mode 100644 packages/backend-common/src/reading/GitlabReaderProcessor.ts diff --git a/packages/backend-common/src/reading/GitlabReaderProcessor.ts b/packages/backend-common/src/reading/GitlabReaderProcessor.ts deleted file mode 100644 index c33b388e1a..0000000000 --- a/packages/backend-common/src/reading/GitlabReaderProcessor.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -export class GitlabReaderProcessor implements LocationProcessor { - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'gitlab') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString()); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - throw result.notFoundError(location, message); - } - } else { - throw result.generalError(location, message); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } - - // Converts - // from: https://gitlab.example.com/a/b/blob/master/c.yaml - // to: https://gitlab.example.com/a/b/raw/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [empty, userOrOrg, repoName, , ...restOfPath] = url.pathname - .split('/') - // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml - .filter(path => path !== '-'); - - if ( - empty !== '' || - userOrOrg === '' || - repoName === '' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitLab URL'); - } - - // Replace 'blob' with 'raw' - url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( - '/', - ); - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 17023d9120..60a8d19db3 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -35,7 +35,7 @@ describe('GitlabUrlReader', () => { }, ]); - it('should build raw api', () => { + it('should build project urls', () => { const processor = new GitlabUrlReader( readConfig(createConfig(undefined))[0], ); @@ -69,7 +69,33 @@ describe('GitlabUrlReader', () => { for (const test of tests) { if (test.url) { - expect(processor.buildRawUrl(test.target, 12345).toString()).toEqual( + expect( + processor.buildProjectUrl(test.target, 12345).toString(), + ).toEqual(test.url.toString()); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should build raw urls', () => { + const processor = new GitlabUrlReader( + readConfig(createConfig(undefined))[0], + ); + + const tests = [ + { + target: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + url: new URL('https://gitlab.example.com/a/b/raw/master/c.yaml'), + err: undefined, + }, + ]; + + for (const test of tests) { + if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( test.url.toString(), ); } else { diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 56ae91bb4b..9931b6d9d5 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -68,8 +68,16 @@ export class GitlabUrlReader implements UrlReader { } async read(url: string): Promise { - const projectID = await this.getProjectID(url); - const builtUrl = this.buildRawUrl(url, projectID); + // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used + // the existence of /~/blob/ to switch the logic. Don't know if this + // makes sense and it might require some more work. + let builtUrl: URL; + if (url.includes('/-/blob/')) { + const projectID = await this.getProjectID(url); + builtUrl = this.buildProjectUrl(url, projectID); + } else { + builtUrl = this.buildRawUrl(url); + } let response: Response; try { @@ -89,9 +97,41 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } + // Converts + // from: https://gitlab.example.com/a/b/blob/master/c.yaml + // to: https://gitlab.example.com/a/b/raw/master/c.yaml + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [empty, userOrOrg, repoName, ...restOfPath] = url.pathname + .split('/') + // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml + .filter(path => path !== '-'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitLab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( + '/', + ); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch - buildRawUrl(target: string, projectID: Number): URL { + buildProjectUrl(target: string, projectID: Number): URL { try { const url = new URL(target); From 07d19aad962dc6402b1cd0ed2a9e95b8d79bbd83 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 16:27:41 +0200 Subject: [PATCH 15/33] backend-common: fix GithubUrlReader tests --- .../src/reading/GithubUrlReader.test.ts | 67 +++++++------------ .../src/reading/GithubUrlReader.ts | 2 +- 2 files changed, 25 insertions(+), 44 deletions(-) diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 513d66c09b..8df464db9f 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,7 +15,6 @@ */ import { ConfigReader } from '@backstage/config'; -import { getVoidLogger } from '../logging'; import { getApiRequestOptions, getApiUrl, @@ -29,7 +28,7 @@ import { describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { it('sets the correct API version', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect((getApiRequestOptions(config).headers as any).Accept).toEqual( 'application/vnd.github.v3.raw', ); @@ -37,12 +36,12 @@ describe('GithubUrlReader', () => { it('inserts a token when needed', () => { const withToken: ProviderConfig = { - target: '', + host: '', apiBaseUrl: '', token: 'A', }; const withoutToken: ProviderConfig = { - target: '', + host: '', apiBaseUrl: '', }; expect( @@ -57,12 +56,12 @@ describe('GithubUrlReader', () => { describe('getRawRequestOptions', () => { it('inserts a token when needed', () => { const withToken: ProviderConfig = { - target: '', + host: '', rawBaseUrl: '', token: 'A', }; const withoutToken: ProviderConfig = { - target: '', + host: '', rawBaseUrl: '', }; expect( @@ -76,13 +75,13 @@ describe('GithubUrlReader', () => { describe('getApiUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { const config: ProviderConfig = { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', }; expect( @@ -109,7 +108,7 @@ describe('GithubUrlReader', () => { it('happy path for ghe', () => { const config: ProviderConfig = { - target: 'https://ghe.mycompany.net', + host: 'ghe.mycompany.net', apiBaseUrl: 'https://ghe.mycompany.net/api/v3', }; expect( @@ -127,13 +126,13 @@ describe('GithubUrlReader', () => { describe('getRawUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { const config: ProviderConfig = { - target: 'https://github.com', + host: 'github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }; expect( @@ -150,7 +149,7 @@ describe('GithubUrlReader', () => { it('happy path for ghe', () => { const config: ProviderConfig = { - target: 'https://ghe.mycompany.net', + host: 'ghe.mycompany.net', rawBaseUrl: 'https://ghe.mycompany.net/raw', }; expect( @@ -166,23 +165,23 @@ describe('GithubUrlReader', () => { describe('readConfig', () => { function config( - providers: { target: string; apiBaseUrl?: string; token?: string }[], + providers: { host: string; apiBaseUrl?: string; token?: string }[], ) { return ConfigReader.fromConfigs([ { context: '', data: { - catalog: { processors: { github: { providers } } }, + integrations: { github: providers }, }, }, ]); } it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([]), getVoidLogger()); + const output = readConfig(config([])); expect(output).toEqual([ { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }, @@ -190,13 +189,10 @@ describe('GithubUrlReader', () => { }); it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig( - config([{ target: 'https://github.com' }]), - getVoidLogger(), - ); + const output = readConfig(config([{ host: 'github.com' }])); expect(output).toEqual([ { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }, @@ -204,34 +200,19 @@ describe('GithubUrlReader', () => { }); it('rejects custom targets with no base URLs', () => { - expect(() => - readConfig( - config([{ target: 'https://ghe.company.com' }]), - getVoidLogger(), - ), - ).toThrow( - 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', + expect(() => readConfig(config([{ host: 'ghe.company.com' }]))).toThrow( + "GitHub integration for 'ghe.company.com' must configure an explicit apiBaseUrl and rawBaseUrl", ); }); it('rejects funky configs', () => { + expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/); + expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/); expect(() => - readConfig(config([{ target: 7 } as any]), getVoidLogger()), - ).toThrow(/target/); - expect(() => - readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), - ).toThrow(/target/); - expect(() => - readConfig( - config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), - getVoidLogger(), - ), + readConfig(config([{ host: 'github.com', apiBaseUrl: 7 } as any])), ).toThrow(/apiBaseUrl/); expect(() => - readConfig( - config([{ target: 'https://github.com', token: 7 } as any]), - getVoidLogger(), - ), + readConfig(config([{ host: 'github.com', token: 7 } as any])), ).toThrow(/token/); }); }); @@ -239,7 +220,7 @@ describe('GithubUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { const processor = new GithubUrlReader({ - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', }); await expect( diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index b8e996fcd9..6902188e17 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -163,7 +163,7 @@ export function readConfig(config: Config): ProviderConfig[] { if (!apiBaseUrl && !rawBaseUrl) { throw new Error( - `GitHub integration for ${host} must configure an explicit apiBaseUrl and rawBaseUrl`, + `GitHub integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`, ); } From 4978f06514eadd94f991be1afb43e909ec7adcdd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:00:38 +0200 Subject: [PATCH 16/33] catalog-backend: make UrlReaderProcessor backwards-compatible with old location types --- .../src/ingestion/LocationReaders.ts | 7 ++++- .../processors/UrlReaderProcessor.ts | 28 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index a217e07f1d..b911a7a50d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -88,7 +88,12 @@ export class LocationReaders implements LocationReader { new BitbucketApiReaderProcessor(config), new AzureApiReaderProcessor(config), GithubOrgReaderProcessor.fromConfig(config), - options.reader ? new UrlReaderProcessor(options.reader) : [], + options.reader + ? new UrlReaderProcessor({ + reader: options.reader, + logger: options.logger, + }) + : [], new YamlProcessor(), PlaceholderProcessor.default(), new CodeOwnersProcessor(), diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 94a7b01336..2bc05bf1cb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -14,25 +14,47 @@ * limitations under the License. */ +import { Logger } from 'winston'; import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +// TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this +// can be removed in a bit +const deprecatedTypes = [ + 'github', + 'github/api', + 'bitbucket/api', + 'gitlab/api', + 'azure/api', +]; + +type Options = { + reader: UrlReader; + logger: Logger; +}; + export class UrlReaderProcessor implements LocationProcessor { - constructor(private readonly reader: UrlReader) {} + constructor(private readonly options: Options) {} async readLocation( location: LocationSpec, optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'url') { + if (deprecatedTypes.includes(location.type)) { + // TODO(Rugvip): Let's not enable this warning yet, as we want to move over the example YAMLs + // in this repo to use the 'url' type first. + // this.options.logger.warn( + // `Using deprecated location type '${location.type}' for '${location.target}', use 'url' instead`, + // ); + } else if (location.type !== 'url') { return false; } try { - const data = await this.reader.read(location.target); + const data = await this.options.reader.read(location.target); emit(result.data(location, data)); } catch (error) { const message = `Unable to read ${location.type}, ${error}`; From 327f87562d337bce4d559f05becbcd035d61ab7c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:14:22 +0200 Subject: [PATCH 17/33] backend: add UrlReader --- packages/backend/src/index.ts | 4 +++- packages/backend/src/plugins/catalog.ts | 5 +++-- packages/backend/src/types.ts | 3 ++- .../templates/default-app/packages/backend/src/index.ts | 4 +++- .../default-app/packages/backend/src/plugins/catalog.ts | 3 ++- .../templates/default-app/packages/backend/src/types.ts | 3 ++- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index e86034dd29..6cfc4bc519 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -32,6 +32,7 @@ import { useHotMemoize, notFoundHandler, SingleHostDiscovery, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; @@ -60,8 +61,9 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); + const reader = UrlReaders.default({ logger }).createWithConfig(config); const discovery = SingleHostDiscovery.fromConfig(config); - return { logger, database, config, discovery }; + return { logger, database, config, reader, discovery }; }; } diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 219b04a0f7..c1d390f8cc 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -28,10 +28,11 @@ import { useHotCleanup } from '@backstage/backend-common'; export default async function createPlugin({ logger, - database, config, + reader, + database, }: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, reader, config }); const db = await DatabaseManager.createDatabase(database, { logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 3709fc8d9a..9257fcc9cf 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -17,11 +17,12 @@ import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; database: Knex; config: Config; + reader: UrlReader; discovery: PluginEndpointDiscovery; }; diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 47ab2c5993..8165b6861a 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -16,6 +16,7 @@ import { useHotMemoize, notFoundHandler, SingleHostDiscovery, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import auth from './plugins/auth'; @@ -38,8 +39,9 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); + const reader = UrlReaders.default({ logger }).createWithConfig(config); const discovery = SingleHostDiscovery.fromConfig(config); - return { logger, database, config, discovery }; + return { logger, database, config, reader, discovery }; }; } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts index aa0538a6d8..345ac4c23d 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -13,9 +13,10 @@ import { useHotCleanup } from '@backstage/backend-common'; export default async function createPlugin({ logger, config, + reader, database, }: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, reader, config }); const db = await DatabaseManager.createDatabase(database, { logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); diff --git a/packages/create-app/templates/default-app/packages/backend/src/types.ts b/packages/create-app/templates/default-app/packages/backend/src/types.ts index d145255390..aa003f63a5 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/types.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/types.ts @@ -1,11 +1,12 @@ import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; database: Knex; config: Config; + reader: UrlReader discovery: PluginEndpointDiscovery; }; From b971534dfbf09050d225fd81d70dd4bc59d7a017 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:21:00 +0200 Subject: [PATCH 18/33] backend-common: added FetchUrlReader and use as fallback by default --- .../src/reading/FetchUrlReader.ts | 43 +++++++++++++++++++ .../src/reading/UrlReaderPredicateMux.ts | 14 ++++++ .../backend-common/src/reading/UrlReaders.ts | 9 +++- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 packages/backend-common/src/reading/FetchUrlReader.ts diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts new file mode 100644 index 0000000000..cdaeb1ba1d --- /dev/null +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch, { Response } from 'node-fetch'; +import { NotFoundError } from '../errors'; +import { UrlReader } from './types'; + +/** + * A UrlReader that does a plain fetch of the URL. + */ +export class FetchUrlReader implements UrlReader { + async read(url: string): Promise { + let response: Response; + try { + response = await fetch(url); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.ok) { + return response.buffer(); + } + + const message = `could not read ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } +} diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index a02c7df86c..8c3f9f17d7 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -23,12 +23,22 @@ export type UrlReaderPredicateTuple = { reader: UrlReader; }; +type Options = { + // UrlReader to fall back to if no other reader is matched + fallback?: UrlReader; +}; + /** * A UrlReader implementation that selects from a set of UrlReaders * based on a predicate tied to each reader. */ export class UrlReaderPredicateMux implements UrlReader { private readonly readers: UrlReaderPredicateTuple[] = []; + private readonly fallback?: UrlReader; + + constructor({ fallback }: Options) { + this.fallback = fallback; + } register(tuple: UrlReaderPredicateTuple): void { this.readers.push(tuple); @@ -43,6 +53,10 @@ export class UrlReaderPredicateMux implements UrlReader { } } + if (this.fallback) { + return this.fallback.read(url); + } + throw new Error(`No reader found that could handle '${url}'`); } } diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2535c953e9..5119fcf560 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -25,6 +25,7 @@ import { AzureUrlReader } from './AzureUrlReader'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; +import { FetchUrlReader } from './FetchUrlReader'; export type ReaderFactoryOptions = { config: Config; @@ -62,12 +63,14 @@ export class UrlReaders { GitlabUrlReader.factory, ], logger, + new FetchUrlReader(), ); } private constructor( private readonly factories: ReaderFactory[], private readonly logger: Logger, + private fallback?: UrlReader, ) {} /** @@ -75,7 +78,7 @@ export class UrlReaders { * reader type needs to have a registered factory, or an error will be thrown. */ createWithConfig(config: Config): UrlReader { - const mux = new UrlReaderPredicateMux(); + const mux = new UrlReaderPredicateMux({ fallback: this.fallback }); const readers = []; for (const factory of this.factories) { @@ -100,4 +103,8 @@ export class UrlReaders { addFactory(factory: ReaderFactory) { this.factories.push(factory); } + + setFallback(reader?: UrlReader) { + this.fallback = reader; + } } From 238715aac2e3cf326b0c82282497d3b1c8e5c8bc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:21:59 +0200 Subject: [PATCH 19/33] config,docs: switch to using url location type --- app-config.yaml | 10 +++++----- docs/features/software-catalog/configuration.md | 4 ++-- docs/features/software-catalog/index.md | 2 +- docs/features/software-catalog/installation.md | 16 ++++++++-------- .../software-templates/adding-templates.md | 2 +- docs/features/software-templates/installation.md | 8 ++++---- .../templates/default-app/app-config.yaml.hbs | 14 +++++++------- .../src/ingestion/CatalogRules.ts | 4 ++-- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index d21e858e8a..ac327f04a9 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -124,19 +124,19 @@ catalog: locations: # Backstage example components - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml # Example component for github-actions - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml # Example component for techdocs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml # Backstage example APIs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml # Backstage example templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml scaffolder: diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index c98bd73ca3..228dcccf48 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -74,7 +74,7 @@ the catalog under the `catalog.locations` key, for example: ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` @@ -97,7 +97,7 @@ catalog: - allow: [Component, API, Location, Template] locations: - - type: github + - type: url target: https://github.com/org/example/blob/master/org-data.yaml rules: - allow: [Group] diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index 8be6a00579..4ba83146db 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -90,7 +90,7 @@ above example can be added using the following configuration: ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` diff --git a/docs/features/software-catalog/installation.md b/docs/features/software-catalog/installation.md index a825020eb6..26990033aa 100644 --- a/docs/features/software-catalog/installation.md +++ b/docs/features/software-catalog/installation.md @@ -166,21 +166,21 @@ our example templates through static configuration. Add the following to the catalog: locations: # Backstage Example Component - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml ``` diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 4283af7c72..5a4e0049ab 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -66,7 +66,7 @@ for example ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml rules: - allow: [Template] diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index eff7e53ad3..08f1882425 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -191,13 +191,13 @@ our example templates through static configuration. Add the following to the catalog: locations: # Backstage Example Templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml ``` diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index f1278bd3c3..a179efe982 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -82,31 +82,31 @@ catalog: # env: GHE_PRIVATE_TOKEN locations: # Backstage example components - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml # Backstage example APIs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml # Backstage example templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml rules: - allow: [Template] diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index e964524bc7..e82699f526 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -74,11 +74,11 @@ export class CatalogRulesEnforcer { * - allow: [Component, API] * * locations: - * - type: github + * - type: url * target: https://github.com/org/repo/blob/master/users.yaml * rules: * - allow: [User, Group] - * - type: github + * - type: url * target: https://github.com/org/repo/blob/master/systems.yaml * rules: * - allow: [System] From 3289598526d28a2f88a4eec2bae2230c7bc3f13e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:22:49 +0200 Subject: [PATCH 20/33] backend-common: remove UrlReaderHostMux --- .../src/reading/UrlReaderHostMux.ts | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 packages/backend-common/src/reading/UrlReaderHostMux.ts diff --git a/packages/backend-common/src/reading/UrlReaderHostMux.ts b/packages/backend-common/src/reading/UrlReaderHostMux.ts deleted file mode 100644 index 446308c54c..0000000000 --- a/packages/backend-common/src/reading/UrlReaderHostMux.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { UrlReader } from './types'; - -/** - * A UrlReader implementation that selects from a set of UrlReaders - * based on the host in the URL. - */ -export class UrlReaderHostMux implements UrlReader { - private readonly readers = new Map(); - - register(host: string, reader: UrlReader): void { - this.readers.set(host, reader); - } - - read(url: string): Promise { - const parsed = new URL(url); - - const reader = this.readers.get(parsed.host); - if (!reader) { - throw new Error(`No reader registered for host ${parsed.host}`); - } - - return reader.read(url); - } -} From 69b3c034734e21b428a4332789b57dc6e62c32d0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:34:51 +0200 Subject: [PATCH 21/33] backend-common: added toString methods for all readers + log in backend instead of at creation --- packages/backend-common/src/reading/AzureUrlReader.ts | 5 +++++ .../backend-common/src/reading/BitbucketUrlReader.ts | 5 +++++ packages/backend-common/src/reading/FetchUrlReader.ts | 4 ++++ packages/backend-common/src/reading/GithubUrlReader.ts | 5 +++++ packages/backend-common/src/reading/GitlabUrlReader.ts | 5 +++++ .../backend-common/src/reading/UrlReaderPredicateMux.ts | 6 ++++++ packages/backend-common/src/reading/UrlReaders.ts | 6 ------ packages/backend/src/index.ts | 9 ++++++--- .../templates/default-app/packages/backend/src/index.ts | 9 ++++++--- 9 files changed, 42 insertions(+), 12 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 202695eaed..ade2cd8793 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -162,4 +162,9 @@ export class AzureUrlReader implements UrlReader { throw new Error(`Incorrect url: ${target}, ${e}`); } } + + toString() { + const { host, token } = this.options; + return `azure{host=${host},authed=${Boolean(token)}}`; + } } diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 62739ded1c..53966471af 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -158,4 +158,9 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(`Incorrect url: ${target}, ${e}`); } } + + toString() { + const { host, auth } = this.options; + return `bitbucket{host=${host},authed=${Boolean(auth)}}`; + } } diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index cdaeb1ba1d..ea56f2182d 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -40,4 +40,8 @@ export class FetchUrlReader implements UrlReader { } throw new Error(message); } + + toString() { + return 'fetch{}'; + } } diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 6902188e17..7a68890b00 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -229,4 +229,9 @@ export class GithubUrlReader implements UrlReader { } throw new Error(message); } + + toString() { + const { host, token } = this.config; + return `github{host=${host},authed=${Boolean(token)}}`; + } } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 9931b6d9d5..5b4708a188 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -187,4 +187,9 @@ export class GitlabUrlReader implements UrlReader { throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); } } + + toString() { + const { host, token } = this.options; + return `gitlab{host=${host},authed=${Boolean(token)}}`; + } } diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 8c3f9f17d7..098fb77edc 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -59,4 +59,10 @@ export class UrlReaderPredicateMux implements UrlReader { throw new Error(`No reader found that could handle '${url}'`); } + + toString() { + return `predicateMux{readers=${this.readers + .map(t => t.reader) + .join(',')},fallback=${this.fallback}}`; + } } diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 5119fcf560..f27bfea0a3 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -79,21 +79,15 @@ export class UrlReaders { */ createWithConfig(config: Config): UrlReader { const mux = new UrlReaderPredicateMux({ fallback: this.fallback }); - const readers = []; for (const factory of this.factories) { const tuples = factory({ config, logger: this.logger }); for (const tuple of tuples) { mux.register(tuple); - readers.push(tuple.reader); } } - this.logger.info( - `Registered the following UrlReaders: ${readers.join(', ')}`, - ); - return mux; } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 6cfc4bc519..ccaedb2830 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -50,9 +50,14 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); + const root = getRootLogger(); + const reader = UrlReaders.default({ logger: root }).createWithConfig(config); + const discovery = SingleHostDiscovery.fromConfig(config); + + root.info(`Created UrlReader ${reader}`); return (plugin: string): PluginEnvironment => { - const logger = getRootLogger().child({ type: 'plugin', plugin }); + const logger = root.child({ type: 'plugin', plugin }); const database = createDatabaseClient( config.getConfig('backend.database'), { @@ -61,8 +66,6 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); - const reader = UrlReaders.default({ logger }).createWithConfig(config); - const discovery = SingleHostDiscovery.fromConfig(config); return { logger, database, config, reader, discovery }; }; } diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 8165b6861a..a87be04d78 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -28,9 +28,14 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); + const root = getRootLogger(); + const reader = UrlReaders.default({ logger: root }).createWithConfig(config); + const discovery = SingleHostDiscovery.fromConfig(config); + + root.info(`Created UrlReader ${reader}`); return (plugin: string): PluginEnvironment => { - const logger = getRootLogger().child({ type: 'plugin', plugin }); + const logger = root.child({ type: 'plugin', plugin }); const database = createDatabaseClient( config.getConfig('backend.database'), { @@ -39,8 +44,6 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); - const reader = UrlReaders.default({ logger }).createWithConfig(config); - const discovery = SingleHostDiscovery.fromConfig(config); return { logger, database, config, reader, discovery }; }; } From 703614b35b50c0afc0da4766c5d33f411184b93c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:43:24 +0200 Subject: [PATCH 22/33] catalog-backend: make reader option required for LocationReaders --- .../catalog-backend/src/ingestion/LocationReaders.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index b911a7a50d..d2a6b2404d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -56,7 +56,7 @@ import { LocationReader, ReadLocationResult } from './types'; const MAX_DEPTH = 10; type Options = { - reader?: UrlReader; + reader: UrlReader; logger?: Logger; config?: Config; processors?: LocationProcessor[]; @@ -72,7 +72,7 @@ export class LocationReaders implements LocationReader { static defaultProcessors(options: { logger: Logger; - reader?: UrlReader; + reader: UrlReader; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { @@ -88,12 +88,7 @@ export class LocationReaders implements LocationReader { new BitbucketApiReaderProcessor(config), new AzureApiReaderProcessor(config), GithubOrgReaderProcessor.fromConfig(config), - options.reader - ? new UrlReaderProcessor({ - reader: options.reader, - logger: options.logger, - }) - : [], + new UrlReaderProcessor(options), new YamlProcessor(), PlaceholderProcessor.default(), new CodeOwnersProcessor(), From 21159258a2218be241188ba74aaa93b237254d8f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:59:28 +0200 Subject: [PATCH 23/33] catalog-backend: added back readers for backwards-compatibility --- .../src/ingestion/LocationReaders.ts | 43 ++- .../processors/AzureApiReaderProcessor.ts | 147 ++++++++++ .../processors/BitbucketApiReaderProcessor.ts | 138 +++++++++ .../processors/GithubReaderProcessor.ts | 274 ++++++++++++++++++ .../processors/GitlabApiReaderProcessor.ts | 141 +++++++++ .../processors/GitlabReaderProcessor.ts | 94 ++++++ 6 files changed, 832 insertions(+), 5 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index d2a6b2404d..96ffaf398c 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,6 +32,7 @@ import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProc import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; +import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; @@ -77,16 +78,48 @@ export class LocationReaders implements LocationReader { entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { + logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; + + // TODO(Rugvip): These are added for backwards compatibility if config exists + // The idea is to have everyone migrate from using the old processors to the new + // integration config driven UrlReaders. In an upcoming release we can then completely + // remove support for the old processors, but still keep handling the deprecated location + // types for a while, but with a warning. + const oldProcessors = []; + const pc = config.getOptionalConfig('catalog.processors'); + if (pc?.has('github')) { + logger.warn( + `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, + ); + oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); + } + if (pc?.has('gitlabApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, + ); + oldProcessors.push(new GitlabApiReaderProcessor(config)); + oldProcessors.push(new GitlabReaderProcessor()); + } + if (pc?.has('bitbucketApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, + ); + oldProcessors.push(new BitbucketApiReaderProcessor(config)); + } + if (pc?.has('azureApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, + ); + oldProcessors.push(new AzureApiReaderProcessor(config)); + } + return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GitlabApiReaderProcessor(config), - new GitlabReaderProcessor(), - new BitbucketApiReaderProcessor(config), - new AzureApiReaderProcessor(config), + ...oldProcessors, GithubOrgReaderProcessor.fromConfig(config), new UrlReaderProcessor(options), new YamlProcessor(), @@ -96,7 +129,7 @@ export class LocationReaders implements LocationReader { new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), new AnnotateLocationEntityProcessor(), - ].flat(); + ]; } constructor({ diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..2a15c18c84 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.azureApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'items', + ].join('/'); + + const queryParams = [`path=${path}`]; + + if (ref) { + queryParams.push(`version=${ref}`); + } + + url.search = queryParams.join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts new file mode 100644 index 0000000000..b3720a5b76 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class BitbucketApiReaderProcessor implements LocationProcessor { + private username: string; + private password: string; + + constructor(config: Config) { + this.username = + config.getOptionalString('catalog.processors.bitbucketApi.username') ?? + ''; + this.password = + config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.username !== '' && this.password !== '') { + headers.Authorization = `Basic ${Buffer.from( + `${this.username}:${this.password}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + srcKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'bitbucket.org' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + srcKeyword !== 'src' + ) { + throw new Error('Wrong Bitbucket URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + '2.0', + 'repositories', + userOrOrg, + repoName, + 'src', + ref, + ...restOfPath, + ].join('/'); + url.hostname = 'api.bitbucket.org'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts new file mode 100644 index 0000000000..89d1f69860 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } + + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + const response = await fetch(url.toString(), options); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts new file mode 100644 index 0000000000..aeba7a7ebe --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class GitlabApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; + if (this.privateToken !== '') { + headers['PRIVATE-TOKEN'] = this.privateToken; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab/api') { + return false; + } + + try { + const projectID = await this.getProjectID(location.target); + const url = this.buildRawUrl(location.target, projectID); + const response = await fetch(url.toString(), this.getRequestOptions()); + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch + buildRawUrl(target: string, projectID: Number): URL { + try { + const url = new URL(target); + + const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + + const [branch, ...filePath] = branchAndfilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + async getProjectID(target: string): Promise { + const url = new URL(target); + + if ( + // absPaths to gitlab files should contain /-/blob + // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + !url.pathname.match(/\/\-\/blob\//) + ) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Find ProjectID from url + // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' + // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + this.getRequestOptions(), + ); + const projectIDJson = await response.json(); + const projectID: Number = projectIDJson.id; + + return projectID; + } catch (e) { + throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts new file mode 100644 index 0000000000..b51bf85a8e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class GitlabReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + throw result.notFoundError(location, message); + } + } else { + throw result.generalError(location, message); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://gitlab.example.com/a/b/blob/master/c.yaml + // to: https://gitlab.example.com/a/b/raw/master/c.yaml + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [empty, userOrOrg, repoName, , ...restOfPath] = url.pathname + .split('/') + // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml + .filter(path => path !== '-'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitLab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( + '/', + ); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} From 027056d93811ab1fc811d77cd23a7ecc1ea0ed8f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 18:40:28 +0200 Subject: [PATCH 24/33] catalog-backend: fix tests for UrlReaderProcessor --- .../processors/UrlReaderProcessor.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index 1ab4cb09b8..e8ee32e765 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -22,6 +22,8 @@ import { } from './types'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost:23000'; @@ -32,7 +34,11 @@ describe('UrlReaderProcessor', () => { afterAll(() => server.close()); it('should load from url', async () => { - const processor = new UrlReaderProcessor(); + const logger = getVoidLogger(); + const reader = UrlReaders.default({ logger }).createWithConfig( + new ConfigReader({}), + ); + const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', target: `${mockApiOrigin}/component.yaml`, @@ -54,7 +60,11 @@ describe('UrlReaderProcessor', () => { }); it('should fail load from url with error', async () => { - const processor = new UrlReaderProcessor(); + const logger = getVoidLogger(); + const reader = UrlReaders.default({ logger }).createWithConfig( + new ConfigReader({}), + ); + const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', target: `${mockApiOrigin}/component-notfound.yaml`, @@ -74,7 +84,7 @@ describe('UrlReaderProcessor', () => { expect(generated.location).toBe(spec); expect(generated.error.name).toBe('NotFoundError'); expect(generated.error.message).toBe( - `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`, + `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`, ); }); }); From caa4ca72c0100589c4af048b309ecc4ca61d3635 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 18:41:25 +0200 Subject: [PATCH 25/33] catalog-backend: update standalone server setup --- plugins/catalog-backend/src/service/standaloneServer.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 0b532d1664..8ee01bb67f 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -17,6 +17,7 @@ import { createServiceBuilder, loadBackendConfig, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { Server } from 'http'; @@ -39,12 +40,13 @@ export async function startStandaloneServer( ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); const config = ConfigReader.fromConfigs(await loadBackendConfig()); + const reader = UrlReaders.default({ logger }).createWithConfig(config); logger.debug('Creating application...'); const db = await DatabaseManager.createInMemoryDatabase({ logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, config, reader }); const higherOrderOperation = new HigherOrderOperations( entitiesCatalog, locationsCatalog, From 77c42b3fd3ea4f2a6791e76554b8d70228d54b51 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 14:53:45 +0200 Subject: [PATCH 26/33] backend-common: refactor AzureUrlReader to test public methods using msw --- packages/backend-common/package.json | 1 + .../src/reading/AzureUrlReader.test.ts | 199 +++++++++--------- .../src/reading/AzureUrlReader.ts | 34 ++- packages/backend-common/src/setupTests.ts | 2 - 4 files changed, 113 insertions(+), 123 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index b2550c6551..08b2cb71a0 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -73,6 +73,7 @@ "http-errors": "^1.7.3", "jest": "^26.0.1", "jest-fetch-mock": "^3.0.3", + "msw": "^0.20.5", "supertest": "^4.0.2" }, "files": [ diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 50db27b314..c9e1fc5bc7 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -14,116 +14,111 @@ * limitations under the License. */ -import { AzureUrlReader, readConfig } from './AzureUrlReader'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { AzureUrlReader } from './AzureUrlReader'; -const host = 'dev.azure.com'; +const logger = getVoidLogger(); describe('AzureUrlReader', () => { - const createConfig = (token: string | undefined) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - integrations: { - azure: [ - { - host, - token, - }, - ], - }, - }, - }, - ]); + const worker = setupServer(); - it('should build raw api', () => { - const processor = new AzureUrlReader( - readConfig(createConfig(undefined))[0], + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), ); - const tests = [ - { - target: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', - url: new URL( - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', - ), - err: undefined, - }, - { - target: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', - url: new URL( - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - ]; + }); + afterEach(() => worker.resetHandlers()); - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { azure: [{ host: 'dev.azure.com', token }] }, + }, + 'test-config', + ); + + it.each([ + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }), + }, + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + headers: expect.objectContaining({ + authorization: 'Basic OjAxMjM0NTY3ODk=', + }), + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig(undefined), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = AzureUrlReader.factory({ config, logger }); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); }); - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - Authorization: 'Basic OjAxMjM0NTY3ODk=', - }, - }, - }, - { - token: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'integrations.azure[0].token' in '', got empty-string, wanted string", - }, - { - token: undefined, - expect: { - headers: {}, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => new AzureUrlReader(readConfig(createConfig(test.token))[0]), - ).toThrowError(test.err); - } else { - const processor = new AzureUrlReader( - readConfig(createConfig(test.token))[0], - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = AzureUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index ade2cd8793..aba4839c34 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -26,7 +26,7 @@ type Options = { token?: string; }; -export function readConfig(config: Config): Options[] { +function readConfig(config: Config): Options[] { const optionsArr = Array(); const providerConfigs = @@ -65,23 +65,6 @@ export class AzureUrlReader implements UrlReader { } } - getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; - - if (this.options.token) { - headers.Authorization = `Basic ${Buffer.from( - `:${this.options.token}`, - 'utf8', - ).toString('base64')}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - async read(url: string): Promise { const builtUrl = this.buildRawUrl(url); @@ -107,7 +90,7 @@ export class AzureUrlReader implements UrlReader { // Converts // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} - buildRawUrl(target: string): URL { + private buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -163,6 +146,19 @@ export class AzureUrlReader implements UrlReader { } } + private getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.options.token) { + headers.Authorization = `Basic ${Buffer.from( + `:${this.options.token}`, + 'utf8', + ).toString('base64')}`; + } + + return { headers }; + } + toString() { const { host, token } = this.options; return `azure{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/setupTests.ts b/packages/backend-common/src/setupTests.ts index f7b6ca962d..ba33cf996b 100644 --- a/packages/backend-common/src/setupTests.ts +++ b/packages/backend-common/src/setupTests.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -require('jest-fetch-mock').enableMocks(); - export {}; From 02eafc70919d8b149457cc91aa1905d4fd319839 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 15:10:34 +0200 Subject: [PATCH 27/33] backend-common: refactor BitbucketUrlReader test and exported symbols --- .../src/reading/BitbucketUrlReader.test.ts | 257 +++++++++--------- .../src/reading/BitbucketUrlReader.ts | 37 ++- 2 files changed, 140 insertions(+), 154 deletions(-) diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 43f797edd4..c3e61fb821 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,151 +14,140 @@ * limitations under the License. */ -import { BitbucketUrlReader, readConfig } from './BitbucketUrlReader'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { BitbucketUrlReader } from './BitbucketUrlReader'; -const host = 'bitbucket.org'; +const logger = getVoidLogger(); describe('BitbucketUrlReader', () => { - const createConfig = ( - username: string | undefined, - appPassword: string | undefined, - ) => - ConfigReader.fromConfigs([ + const worker = setupServer(); + + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); + }); + afterEach(() => worker.resetHandlers()); + + const createConfig = (username?: string, appPassword?: string) => + new ConfigReader( { - context: '', - data: { - integrations: { - bitbucket: [ - { - host, - username: username, - appPassword: appPassword, - }, - ], - }, + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: username, + appPassword: appPassword, + }, + ], }, }, - ]); + 'test-config', + ); - it('should build raw api', () => { - const processor = new BitbucketUrlReader({ host }); - - const tests = [ - { - target: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - url: new URL( + it.each([ + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - ]; + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig('some-user', 'my-secret'), + response: expect.objectContaining({ + headers: expect.objectContaining({ + authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', + }), + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(undefined, 'only-password-provided'), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); }); - it('should return request options', () => { - const tests = [ - { - username: '', - password: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'integrations.bitbucket[0].username' in '', got empty-string, wanted string", - }, - { - username: 'only-user-provided', - password: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in '', got empty-string, wanted string", - }, - { - username: '', - password: 'only-password-provided', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'integrations.bitbucket[0].username' in '', got empty-string, wanted string", - }, - { - username: 'some-user', - password: 'my-secret', - expect: { - headers: { - Authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', - }, - }, - }, - { - username: undefined, - password: undefined, - expect: { - headers: {}, - }, - }, - { - username: 'only-user-provided', - password: undefined, - expect: { - headers: {}, - }, - err: - "Missing required config value at 'integrations.bitbucket[0].appPassword'", - }, - { - username: undefined, - password: 'only-password-provided', - expect: { - headers: {}, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => - new BitbucketUrlReader( - readConfig(createConfig(test.username, test.password))[0], - ), - ).toThrowError(test.err); - } else { - const processor = new BitbucketUrlReader( - readConfig(createConfig(test.username, test.password))[0], - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + url: '', + config: createConfig('', ''), + error: + "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('only-user-provided', ''), + error: + "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('', 'only-password-provided'), + error: + "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('only-user-provided', undefined), + error: + "Missing required config value at 'integrations.bitbucket[0].appPassword'", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 53966471af..3b5301af59 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -29,7 +29,7 @@ type Options = { }; }; -export function readConfig(config: Config): Options[] { +function readConfig(config: Config): Options[] { const optionsArr = Array(); const providerConfigs = @@ -74,23 +74,6 @@ export class BitbucketUrlReader implements UrlReader { } } - getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; - - if (this.options.auth) { - headers.Authorization = `Basic ${Buffer.from( - `${this.options.auth.username}:${this.options.auth.appPassword}`, - 'utf8', - ).toString('base64')}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - async read(url: string): Promise { const builtUrl = this.buildRawUrl(url); @@ -115,8 +98,7 @@ export class BitbucketUrlReader implements UrlReader { // Converts // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml - - buildRawUrl(target: string): URL { + private buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -159,6 +141,21 @@ export class BitbucketUrlReader implements UrlReader { } } + private getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.options.auth) { + headers.Authorization = `Basic ${Buffer.from( + `${this.options.auth.username}:${this.options.auth.appPassword}`, + 'utf8', + ).toString('base64')}`; + } + + return { + headers, + }; + } + toString() { const { host, auth } = this.options; return `bitbucket{host=${host},authed=${Boolean(auth)}}`; From d3265666af5fdf01235f022828440bb874c0764e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 16:18:31 +0200 Subject: [PATCH 28/33] backend-common: refactor GitlabUrlReader test and restore raw url functionality --- .../src/reading/GitlabUrlReader.test.ts | 214 ++++++++---------- .../src/reading/GitlabUrlReader.ts | 36 +-- 2 files changed, 112 insertions(+), 138 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 60a8d19db3..09da9c4e0a 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -14,139 +14,109 @@ * limitations under the License. */ -import { GitlabUrlReader, readConfig } from './GitlabUrlReader'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { GitlabUrlReader } from './GitlabUrlReader'; + +const logger = getVoidLogger(); describe('GitlabUrlReader', () => { - const createConfig = (token: string | undefined) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - integrations: { - gitlab: [ - { - host: 'gitlab.com', - token: token, - }, - ], - }, - }, - }, - ]); + const worker = setupServer(); - it('should build project urls', () => { - const processor = new GitlabUrlReader( - readConfig(createConfig(undefined))[0], + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); + }); + afterEach(() => worker.resetHandlers()); + + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { gitlab: [{ host: 'gitlab.com', token }] }, + }, + 'test-config', ); - const tests = [ - { - target: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - url: new URL( + it.each([ + // Project URLs + { + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - { - target: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - url: new URL( + headers: expect.objectContaining({ + 'private-token': '', + }), + }), + }, + { + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - { - target: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup - url: new URL( + headers: expect.objectContaining({ + 'private-token': '0123456789', + }), + }), + }, + { + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - ]; + }), + }, - for (const test of tests) { - if (test.url) { - expect( - processor.buildProjectUrl(test.target, 12345).toString(), - ).toEqual(test.url.toString()); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } + // Raw URLs + { + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); }); - it('should build raw urls', () => { - const processor = new GitlabUrlReader( - readConfig(createConfig(undefined))[0], - ); - - const tests = [ - { - target: 'https://gitlab.example.com/a/b/blob/master/c.yaml', - url: new URL('https://gitlab.example.com/a/b/raw/master/c.yaml'), - err: undefined, - }, - ]; - - for (const test of tests) { - if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } - }); - - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - 'PRIVATE-TOKEN': '0123456789', - }, - }, - }, - { - token: '', - err: - "Invalid type in config for key 'integrations.gitlab[0].token' in '', got empty-string, wanted string", - expect: { - headers: { - 'PRIVATE-TOKEN': '', - }, - }, - }, - { - token: undefined, - expect: { - headers: { - 'PRIVATE-TOKEN': '', - }, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => new GitlabUrlReader(readConfig(createConfig(test.token))[0]), - ).toThrowError(test.err); - } else { - const processor = new GitlabUrlReader( - readConfig(createConfig(test.token))[0], - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } + it.each([ + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 5b4708a188..a6ffef0a34 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -26,7 +26,7 @@ type Options = { token?: string; }; -export function readConfig(config: Config): Options[] { +function readConfig(config: Config): Options[] { const optionsArr = Array(); const providerConfigs = @@ -59,14 +59,6 @@ export class GitlabUrlReader implements UrlReader { constructor(private readonly options: Options) {} - getRequestOptions(): RequestInit { - return { - headers: { - ['PRIVATE-TOKEN']: this.options.token ?? '', - }, - }; - } - async read(url: string): Promise { // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used // the existence of /~/blob/ to switch the logic. Don't know if this @@ -100,19 +92,23 @@ export class GitlabUrlReader implements UrlReader { // Converts // from: https://gitlab.example.com/a/b/blob/master/c.yaml // to: https://gitlab.example.com/a/b/raw/master/c.yaml - buildRawUrl(target: string): URL { + private buildRawUrl(target: string): URL { try { const url = new URL(target); - const [empty, userOrOrg, repoName, ...restOfPath] = url.pathname - .split('/') - // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml - .filter(path => path !== '-'); + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); if ( empty !== '' || userOrOrg === '' || repoName === '' || + blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { throw new Error('Wrong GitLab URL'); @@ -131,7 +127,7 @@ export class GitlabUrlReader implements UrlReader { // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch - buildProjectUrl(target: string, projectID: Number): URL { + private buildProjectUrl(target: string, projectID: Number): URL { try { const url = new URL(target); @@ -154,7 +150,7 @@ export class GitlabUrlReader implements UrlReader { } } - async getProjectID(target: string): Promise { + private async getProjectID(target: string): Promise { const url = new URL(target); if ( @@ -188,6 +184,14 @@ export class GitlabUrlReader implements UrlReader { } } + private getRequestOptions(): RequestInit { + return { + headers: { + ['PRIVATE-TOKEN']: this.options.token ?? '', + }, + }; + } + toString() { const { host, token } = this.options; return `gitlab{host=${host},authed=${Boolean(token)}}`; From e6ac72b59ef309180a51651cce5539c237fb4f75 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 16:19:07 +0200 Subject: [PATCH 29/33] backend-common: we support reading from more than just gitlab.com --- packages/backend-common/src/reading/GitlabUrlReader.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index a6ffef0a34..13c4e28b07 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -21,7 +21,6 @@ import { UrlReader } from './types'; import { ReaderFactory } from './UrlReaders'; type Options = { - // TODO: added here for future support, but we only allow bitbucket.org for now host: string; token?: string; }; From e6a6f2f264fe542a11cc82f5b6217ec589761a9c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 16:39:36 +0200 Subject: [PATCH 30/33] backend-common: refactor reader types --- .../src/reading/AzureUrlReader.ts | 3 +-- .../src/reading/BitbucketUrlReader.ts | 3 +-- .../src/reading/GithubUrlReader.ts | 3 +-- .../src/reading/GitlabUrlReader.ts | 3 +-- .../src/reading/UrlReaderPredicateMux.ts | 9 +-------- .../backend-common/src/reading/UrlReaders.ts | 20 ++----------------- packages/backend-common/src/reading/types.ts | 17 ++++++++++++++++ 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index aba4839c34..28fbf25eea 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -17,8 +17,7 @@ import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; -import { UrlReader } from './types'; -import { ReaderFactory } from './UrlReaders'; +import { ReaderFactory, UrlReader } from './types'; type Options = { // TODO: added here for future support, but we only allow dev.azure.com for now diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 3b5301af59..e2576eed47 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -16,8 +16,7 @@ import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; -import { ReaderFactory } from './UrlReaders'; -import { UrlReader } from './types'; +import { ReaderFactory, UrlReader } from './types'; import { NotFoundError } from '../errors'; type Options = { diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 7a68890b00..e5bed6dd26 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -18,8 +18,7 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch'; import { NotFoundError } from '../errors'; -import { UrlReader } from './types'; -import { ReaderFactory } from './UrlReaders'; +import { ReaderFactory, UrlReader } from './types'; /** * The configuration parameters for a single GitHub API provider. diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 13c4e28b07..0e430d650c 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -17,8 +17,7 @@ import fetch, { RequestInit, Response } from 'node-fetch'; import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; -import { UrlReader } from './types'; -import { ReaderFactory } from './UrlReaders'; +import { ReaderFactory, UrlReader } from './types'; type Options = { host: string; diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 098fb77edc..7654cc8aac 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -14,14 +14,7 @@ * limitations under the License. */ -import { UrlReader } from './types'; - -export type UrlReaderPredicate = (url: URL) => boolean; - -export type UrlReaderPredicateTuple = { - predicate: UrlReaderPredicate; - reader: UrlReader; -}; +import { UrlReader, UrlReaderPredicateTuple } from './types'; type Options = { // UrlReader to fall back to if no other reader is matched diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index f27bfea0a3..89bd672ce7 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -16,30 +16,14 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { UrlReader } from './types'; -import { - UrlReaderPredicateMux, - UrlReaderPredicateTuple, -} from './UrlReaderPredicateMux'; +import { ReaderFactory, UrlReader } from './types'; +import { UrlReaderPredicateMux } from './UrlReaderPredicateMux'; import { AzureUrlReader } from './AzureUrlReader'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; import { FetchUrlReader } from './FetchUrlReader'; -export type ReaderFactoryOptions = { - config: Config; - logger: Logger; -}; - -/** - * A factory function that can read config to construct zero or more - * UrlReaders along with a predicate for when it should be used. - */ -export type ReaderFactory = ( - options: ReaderFactoryOptions, -) => UrlReaderPredicateTuple[]; - /** * UrlReaders provide various utilities related to the UrlReader interface. */ diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 9b29d1c45d..e423db3eca 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -14,9 +14,26 @@ * limitations under the License. */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + /** * A generic interface for fetching plain data from URLs. */ export type UrlReader = { read(url: string): Promise; }; + +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = (options: { + config: Config; + logger: Logger; +}) => UrlReaderPredicateTuple[]; From 2527afac9e13b22da899e58526b51e1058b9ddfb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 16:52:31 +0200 Subject: [PATCH 31/33] backend-common: refactor UrlReaders to only expose static methods --- .../backend-common/src/reading/UrlReaders.ts | 80 +++++++++---------- packages/backend/src/index.ts | 2 +- .../default-app/packages/backend/src/index.ts | 2 +- .../processors/UrlReaderProcessor.test.ts | 8 +- .../src/service/standaloneServer.ts | 2 +- 5 files changed, 43 insertions(+), 51 deletions(-) diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 89bd672ce7..e1d99a2c49 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -24,48 +24,34 @@ import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; import { FetchUrlReader } from './FetchUrlReader'; +type CreateOptions = { + /** Root config object */ + config: Config; + /** Logger used by all the readers */ + logger: Logger; + /** A list of factories used to construct individual readers that match on URLs */ + factories?: ReaderFactory[]; + /** Fallback reader to use if none of the readers created by the factories match */ + fallback?: UrlReader; +}; + /** * UrlReaders provide various utilities related to the UrlReader interface. */ export class UrlReaders { /** - * Creates a new UrlReaders instance without any known types. + * Creates a UrlReader without any known types. */ - static empty({ logger }: { logger: Logger }) { - return new UrlReaders([], logger); - } + static create({ + logger, + config, + factories, + fallback, + }: CreateOptions): UrlReader { + const mux = new UrlReaderPredicateMux({ fallback: fallback }); - /** - * Creates a new UrlReaders instance that includes all the default factories from this package - */ - static default({ logger }: { logger: Logger }) { - return new UrlReaders( - [ - AzureUrlReader.factory, - BitbucketUrlReader.factory, - GithubUrlReader.factory, - GitlabUrlReader.factory, - ], - logger, - new FetchUrlReader(), - ); - } - - private constructor( - private readonly factories: ReaderFactory[], - private readonly logger: Logger, - private fallback?: UrlReader, - ) {} - - /** - * Constructs a new UrlReader using the provided configuration. Any encountered - * reader type needs to have a registered factory, or an error will be thrown. - */ - createWithConfig(config: Config): UrlReader { - const mux = new UrlReaderPredicateMux({ fallback: this.fallback }); - - for (const factory of this.factories) { - const tuples = factory({ config, logger: this.logger }); + for (const factory of factories ?? []) { + const tuples = factory({ config, logger: logger }); for (const tuple of tuples) { mux.register(tuple); @@ -76,13 +62,23 @@ export class UrlReaders { } /** - * Register a UrlReader factory + * Creates a UrlReader that includes all the default factories from this package. + * + * Any additional factories passed will be loaded before the default ones. + * + * If no fallback reader is passed, a plain fetch reader will be used. */ - addFactory(factory: ReaderFactory) { - this.factories.push(factory); - } - - setFallback(reader?: UrlReader) { - this.fallback = reader; + static default({ logger, config, factories = [], fallback }: CreateOptions) { + return UrlReaders.create({ + logger, + config, + factories: factories.concat([ + AzureUrlReader.factory, + BitbucketUrlReader.factory, + GithubUrlReader.factory, + GitlabUrlReader.factory, + ]), + fallback: fallback ?? new FetchUrlReader(), + }); } } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index ccaedb2830..959af71427 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -51,7 +51,7 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); const root = getRootLogger(); - const reader = UrlReaders.default({ logger: root }).createWithConfig(config); + const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index a87be04d78..684db8aab8 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -29,7 +29,7 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); const root = getRootLogger(); - const reader = UrlReaders.default({ logger: root }).createWithConfig(config); + const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index e8ee32e765..009872cda3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -35,9 +35,7 @@ describe('UrlReaderProcessor', () => { it('should load from url', async () => { const logger = getVoidLogger(); - const reader = UrlReaders.default({ logger }).createWithConfig( - new ConfigReader({}), - ); + const reader = UrlReaders.default({ logger, config: new ConfigReader({}) }); const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', @@ -61,9 +59,7 @@ describe('UrlReaderProcessor', () => { it('should fail load from url with error', async () => { const logger = getVoidLogger(); - const reader = UrlReaders.default({ logger }).createWithConfig( - new ConfigReader({}), - ); + const reader = UrlReaders.default({ logger, config: new ConfigReader({}) }); const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 8ee01bb67f..5939c7eb58 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -40,7 +40,7 @@ export async function startStandaloneServer( ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); const config = ConfigReader.fromConfigs(await loadBackendConfig()); - const reader = UrlReaders.default({ logger }).createWithConfig(config); + const reader = UrlReaders.default({ logger, config }); logger.debug('Creating application...'); const db = await DatabaseManager.createInMemoryDatabase({ logger }); From 440a17b39e0ec8ca5b641cbeec5005fcbbcdde8c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 19:08:07 +0200 Subject: [PATCH 32/33] changeset: add changesets for UrlReader addition --- .changeset/create-app-url-reader-update.md | 6 ++++++ .changeset/new-url-reader.md | 12 ++++++++++++ .changeset/url-reader-processor.md | 5 +++++ 3 files changed, 23 insertions(+) create mode 100644 .changeset/create-app-url-reader-update.md create mode 100644 .changeset/new-url-reader.md create mode 100644 .changeset/url-reader-processor.md diff --git a/.changeset/create-app-url-reader-update.md b/.changeset/create-app-url-reader-update.md new file mode 100644 index 0000000000..eb2e538b77 --- /dev/null +++ b/.changeset/create-app-url-reader-update.md @@ -0,0 +1,6 @@ +--- +'example-backend': patch +'@backstage/create-app': patch +--- + +Bump @backstage/catalog-backend and pass the now required UrlReader interface to the plugin diff --git a/.changeset/new-url-reader.md b/.changeset/new-url-reader.md new file mode 100644 index 0000000000..94758c5bc1 --- /dev/null +++ b/.changeset/new-url-reader.md @@ -0,0 +1,12 @@ +--- +'@backstage/backend-common': patch +--- + +Added new UrlReader interface for reading opaque data from URLs with different providers. + +This new URL reading system is intended as a replacement for the various integrations towards +external systems in the catalog, scaffolder, and techdocs. It is configured via a new top-level +config section called 'integrations'. + +Along with the UrlReader interface is a new UrlReaders class, which exposes static factory +methods for instantiating readers that can read from many different integrations simultaneously. diff --git a/.changeset/url-reader-processor.md b/.changeset/url-reader-processor.md new file mode 100644 index 0000000000..40fef3c106 --- /dev/null +++ b/.changeset/url-reader-processor.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +The catalog backend UrlReaderProcessor now uses a UrlReader from @backstage/backend-common, which must now be supplied to the constructor. From d0c8f72a66021e45fbd9b9597689fa635c3015dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 3 Oct 2020 19:31:27 +0200 Subject: [PATCH 33/33] docs/features/software-catalog: repurpose github processor docs as initial docs for integrations --- .../software-catalog/configuration.md | 68 ++++++++++++------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 228dcccf48..ba5e36ab5d 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -11,36 +11,56 @@ tasks, such as reading raw entity data from a remote source, parsing it, transforming it, and validating it. These processors are configured under the `catalog.processors` key. -### Processor: github +### Processor: url -The `github` processor is responsible for fetching entity data from files on -GitHub or GitHub Enterprise. The configuration for this processor lives under -`catalog.processors.github`. Example: +The `url` processor is responsible for fetching entity data from files in any +external provider like GitHub, GitLab, Bitbucket, etc. The configuration of this +processor lives under the top-level `integrations` key, as it is used by other +parts of Backstage too. ```yaml -catalog: - processors: - github: - providers: - - target: https://github.com - token: - $secret: - env: GITHUB_PRIVATE_TOKEN - - target: https://ghe.example.net - apiBaseUrl: https://ghe.example.net/api/v3 - rawBaseUrl: https://ghe.example.net/raw - token: - $secret: - env: GHE_PRIVATE_TOKEN +integrations: + github: + - host: github.com + token: + $secret: + env: GITHUB_TOKEN + - host: ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_TOKEN + gitlab: + - host: gitlab.com + token: + $secret: + env: GITLAB_TOKEN + bitbucket: + - host: bitbucket.org + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + azure: + - host: dev.azure.com + token: + $secret: + env: AZURE_TOKEN ``` -The main subkey is `providers`, where you can list the various GitHub compatible -providers you want to be able to fetch data from. Each entry is a structure with -up to four elements: +Each key under `integrations` is a separate configuration for each external +provider. The providers each have their own configuration, so let's look at the +GitHub section as an example. -- `target` (required): The string prefix of the location target that you want to - match on, with no trailing slash. For GitHub, it should be exactly - `https://github.com`. +Directly under the `github` key is a list of provider configurations, where you +can list the various GitHub compatible providers you want to be able to fetch +data from. Each entry is a structure with up to four elements: + +- `host` (optional): The host of the location target that you want to match on. + The default host is `github.com`. - `token` (optional): An authentication token as expected by GitHub. If supplied, it will be passed along with all calls to this provider, both API and raw. If it is not supplied, anonymous access will be used.