diff --git a/.changeset/create-app-url-reader-update.md b/.changeset/create-app-url-reader-update.md new file mode 100644 index 0000000000..eb2e538b77 --- /dev/null +++ b/.changeset/create-app-url-reader-update.md @@ -0,0 +1,6 @@ +--- +'example-backend': patch +'@backstage/create-app': patch +--- + +Bump @backstage/catalog-backend and pass the now required UrlReader interface to the plugin diff --git a/.changeset/new-url-reader.md b/.changeset/new-url-reader.md new file mode 100644 index 0000000000..94758c5bc1 --- /dev/null +++ b/.changeset/new-url-reader.md @@ -0,0 +1,12 @@ +--- +'@backstage/backend-common': patch +--- + +Added new UrlReader interface for reading opaque data from URLs with different providers. + +This new URL reading system is intended as a replacement for the various integrations towards +external systems in the catalog, scaffolder, and techdocs. It is configured via a new top-level +config section called 'integrations'. + +Along with the UrlReader interface is a new UrlReaders class, which exposes static factory +methods for instantiating readers that can read from many different integrations simultaneously. diff --git a/.changeset/url-reader-processor.md b/.changeset/url-reader-processor.md new file mode 100644 index 0000000000..40fef3c106 --- /dev/null +++ b/.changeset/url-reader-processor.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +The catalog backend UrlReaderProcessor now uses a UrlReader from @backstage/backend-common, which must now be supplied to the constructor. diff --git a/app-config.yaml b/app-config.yaml index 88df40813e..ac327f04a9 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -67,28 +67,48 @@ kubernetes: clusterLocatorMethod: 'configMultiTenant' clusters: [] +integrations: + github: + - host: github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + ### Example for how to add your GitHub Enterprise instance using the API: + # - host: ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - host: ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN + gitlab: + - host: gitlab.com + token: + $secret: + env: GITLAB_PRIVATE_TOKEN + bitbucket: + - host: bitbucket.org + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + azure: + - host: dev.azure.com + token: + $secret: + env: AZURE_PRIVATE_TOKEN + catalog: rules: - allow: [Component, API, Group, User, Template, Location] + processors: - github: - providers: - - target: https://github.com - token: - $secret: - env: GITHUB_PRIVATE_TOKEN - #### Example for how to add your GitHub Enterprise instance using the API: - # - target: https://ghe.example.net - # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN - #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): - # - target: https://ghe.example.net - # rawBaseUrl: https://ghe.example.net/raw - # token: - # $secret: - # env: GHE_PRIVATE_TOKEN githubOrg: providers: - target: https://github.com @@ -101,37 +121,22 @@ catalog: # token: # $secret: # env: GHE_PRIVATE_TOKEN - bitbucketApi: - username: - $secret: - env: BITBUCKET_USERNAME - appPassword: - $secret: - env: BITBUCKET_APP_PASSWORD - gitlabApi: - privateToken: - $secret: - env: GITLAB_PRIVATE_TOKEN - azureApi: - privateToken: - $secret: - env: AZURE_PRIVATE_TOKEN locations: # Backstage example components - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml # Example component for github-actions - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml # Example component for techdocs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml # Backstage example APIs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml # Backstage example templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml scaffolder: diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index c98bd73ca3..ba5e36ab5d 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -11,36 +11,56 @@ tasks, such as reading raw entity data from a remote source, parsing it, transforming it, and validating it. These processors are configured under the `catalog.processors` key. -### Processor: github +### Processor: url -The `github` processor is responsible for fetching entity data from files on -GitHub or GitHub Enterprise. The configuration for this processor lives under -`catalog.processors.github`. Example: +The `url` processor is responsible for fetching entity data from files in any +external provider like GitHub, GitLab, Bitbucket, etc. The configuration of this +processor lives under the top-level `integrations` key, as it is used by other +parts of Backstage too. ```yaml -catalog: - processors: - github: - providers: - - target: https://github.com - token: - $secret: - env: GITHUB_PRIVATE_TOKEN - - target: https://ghe.example.net - apiBaseUrl: https://ghe.example.net/api/v3 - rawBaseUrl: https://ghe.example.net/raw - token: - $secret: - env: GHE_PRIVATE_TOKEN +integrations: + github: + - host: github.com + token: + $secret: + env: GITHUB_TOKEN + - host: ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: + $secret: + env: GHE_TOKEN + gitlab: + - host: gitlab.com + token: + $secret: + env: GITLAB_TOKEN + bitbucket: + - host: bitbucket.org + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + azure: + - host: dev.azure.com + token: + $secret: + env: AZURE_TOKEN ``` -The main subkey is `providers`, where you can list the various GitHub compatible -providers you want to be able to fetch data from. Each entry is a structure with -up to four elements: +Each key under `integrations` is a separate configuration for each external +provider. The providers each have their own configuration, so let's look at the +GitHub section as an example. -- `target` (required): The string prefix of the location target that you want to - match on, with no trailing slash. For GitHub, it should be exactly - `https://github.com`. +Directly under the `github` key is a list of provider configurations, where you +can list the various GitHub compatible providers you want to be able to fetch +data from. Each entry is a structure with up to four elements: + +- `host` (optional): The host of the location target that you want to match on. + The default host is `github.com`. - `token` (optional): An authentication token as expected by GitHub. If supplied, it will be passed along with all calls to this provider, both API and raw. If it is not supplied, anonymous access will be used. @@ -74,7 +94,7 @@ the catalog under the `catalog.locations` key, for example: ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` @@ -97,7 +117,7 @@ catalog: - allow: [Component, API, Location, Template] locations: - - type: github + - type: url target: https://github.com/org/example/blob/master/org-data.yaml rules: - allow: [Group] diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index 8be6a00579..4ba83146db 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -90,7 +90,7 @@ above example can be added using the following configuration: ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` diff --git a/docs/features/software-catalog/installation.md b/docs/features/software-catalog/installation.md index a825020eb6..26990033aa 100644 --- a/docs/features/software-catalog/installation.md +++ b/docs/features/software-catalog/installation.md @@ -166,21 +166,21 @@ our example templates through static configuration. Add the following to the catalog: locations: # Backstage Example Component - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml ``` diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 4283af7c72..5a4e0049ab 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -66,7 +66,7 @@ for example ```yaml catalog: locations: - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml rules: - allow: [Template] diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index eff7e53ad3..08f1882425 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -191,13 +191,13 @@ our example templates through static configuration. Add the following to the catalog: locations: # Backstage Example Templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml ``` diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 72f047761d..08b2cb71a0 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -39,11 +39,13 @@ "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", + "git-url-parse": "^11.2.0", "helmet": "^4.0.0", "knex": "^0.21.1", "lodash": "^4.17.15", "logform": "^2.1.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.0", "prom-client": "^12.0.0", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", @@ -62,6 +64,7 @@ "@types/compression": "^1.7.0", "@types/http-errors": "^1.6.3", "@types/morgan": "^1.9.0", + "@types/node-fetch": "^2.5.7", "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", "@types/webpack-env": "^1.15.2", @@ -70,6 +73,7 @@ "http-errors": "^1.7.3", "jest": "^26.0.1", "jest-fetch-mock": "^3.0.3", + "msw": "^0.20.5", "supertest": "^4.0.2" }, "files": [ diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 7074040655..e968edef69 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -20,6 +20,7 @@ export * from './discovery'; export * from './errors'; export * from './logging'; export * from './middleware'; +export * from './reading'; export * from './service'; export * from './paths'; export * from './hot'; diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts new file mode 100644 index 0000000000..c9e1fc5bc7 --- /dev/null +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { AzureUrlReader } from './AzureUrlReader'; + +const logger = getVoidLogger(); + +describe('AzureUrlReader', () => { + const worker = setupServer(); + + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); + }); + afterEach(() => worker.resetHandlers()); + + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { azure: [{ host: 'dev.azure.com', token }] }, + }, + 'test-config', + ); + + it.each([ + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }), + }, + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + headers: expect.objectContaining({ + authorization: 'Basic OjAxMjM0NTY3ODk=', + }), + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig(undefined), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = AzureUrlReader.factory({ config, logger }); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = AzureUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); + }); +}); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts new file mode 100644 index 0000000000..28fbf25eea --- /dev/null +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -0,0 +1,165 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; +import { Config } from '@backstage/config'; +import { NotFoundError } from '../errors'; +import { ReaderFactory, UrlReader } from './types'; + +type Options = { + // TODO: added here for future support, but we only allow dev.azure.com for now + host: string; + token?: string; +}; + +function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.azure') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'dev.azure.com'; + const token = providerConfig.getOptionalString('token'); + + optionsArr.push({ host, token }); + } + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public azure repos. + if (!optionsArr.some(p => p.host === 'dev.azure.com')) { + optionsArr.push({ host: 'dev.azure.com' }); + } + + return optionsArr; +} + +export class AzureUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new AzureUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) { + if (options.host !== 'dev.azure.com') { + throw Error( + `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, + ); + } + } + + async read(url: string): Promise { + const builtUrl = this.buildRawUrl(url); + + let response: Response; + try { + response = await fetch(builtUrl.toString(), this.getRequestOptions()); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { + return response.buffer(); + } + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'items', + ].join('/'); + + const queryParams = [`path=${path}`]; + + if (ref) { + queryParams.push(`version=${ref}`); + } + + url.search = queryParams.join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + private getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.options.token) { + headers.Authorization = `Basic ${Buffer.from( + `:${this.options.token}`, + 'utf8', + ).toString('base64')}`; + } + + return { headers }; + } + + toString() { + const { host, token } = this.options; + return `azure{host=${host},authed=${Boolean(token)}}`; + } +} diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts new file mode 100644 index 0000000000..c3e61fb821 --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -0,0 +1,153 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { BitbucketUrlReader } from './BitbucketUrlReader'; + +const logger = getVoidLogger(); + +describe('BitbucketUrlReader', () => { + const worker = setupServer(); + + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); + }); + afterEach(() => worker.resetHandlers()); + + const createConfig = (username?: string, appPassword?: string) => + new ConfigReader( + { + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: username, + appPassword: appPassword, + }, + ], + }, + }, + 'test-config', + ); + + it.each([ + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig('some-user', 'my-secret'), + response: expect.objectContaining({ + headers: expect.objectContaining({ + authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', + }), + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + { + url: + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config: createConfig(undefined, 'only-password-provided'), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + url: '', + config: createConfig('', ''), + error: + "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('only-user-provided', ''), + error: + "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('', 'only-password-provided'), + error: + "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", + }, + { + url: '', + config: createConfig('only-user-provided', undefined), + error: + "Missing required config value at 'integrations.bitbucket[0].appPassword'", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); + }); +}); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts new file mode 100644 index 0000000000..e2576eed47 --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -0,0 +1,162 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch'; +import { Config } from '@backstage/config'; +import { ReaderFactory, UrlReader } from './types'; +import { NotFoundError } from '../errors'; + +type Options = { + // TODO: added here for future support, but we only allow bitbucket.org for now + host: string; + auth?: { + username: string; + appPassword: string; + }; +}; + +function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.bitbucket') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org'; + + let auth; + if (providerConfig.has('username')) { + const username = providerConfig.getString('username'); + const appPassword = providerConfig.getString('appPassword'); + auth = { username, appPassword }; + } + + optionsArr.push({ host, auth }); + } + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public bitbucket repos. + if (!optionsArr.some(p => p.host === 'bitbucket.org')) { + optionsArr.push({ host: 'bitbucket.org' }); + } + + return optionsArr; +} + +export class BitbucketUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new BitbucketUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) { + if (options.host !== 'bitbucket.org') { + throw Error( + `Bitbucket integration currently only supports 'bitbucket.org', tried to use host '${options.host}'`, + ); + } + } + + async read(url: string): Promise { + const builtUrl = this.buildRawUrl(url); + + let response: Response; + try { + response = await fetch(builtUrl.toString(), this.getRequestOptions()); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.ok) { + return response.buffer(); + } + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + // Converts + // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + srcKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'bitbucket.org' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + srcKeyword !== 'src' + ) { + throw new Error('Wrong Bitbucket URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + '2.0', + 'repositories', + userOrOrg, + repoName, + 'src', + ref, + ...restOfPath, + ].join('/'); + url.hostname = 'api.bitbucket.org'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + private getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.options.auth) { + headers.Authorization = `Basic ${Buffer.from( + `${this.options.auth.username}:${this.options.auth.appPassword}`, + 'utf8', + ).toString('base64')}`; + } + + return { + headers, + }; + } + + toString() { + const { host, auth } = this.options; + return `bitbucket{host=${host},authed=${Boolean(auth)}}`; + } +} diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts new file mode 100644 index 0000000000..ea56f2182d --- /dev/null +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch, { Response } from 'node-fetch'; +import { NotFoundError } from '../errors'; +import { UrlReader } from './types'; + +/** + * A UrlReader that does a plain fetch of the URL. + */ +export class FetchUrlReader implements UrlReader { + async read(url: string): Promise { + let response: Response; + try { + response = await fetch(url); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.ok) { + return response.buffer(); + } + + const message = `could not read ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + toString() { + return 'fetch{}'; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts similarity index 66% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts rename to packages/backend-common/src/reading/GithubUrlReader.test.ts index 8eb46db967..8df464db9f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -14,23 +14,21 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; -import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { getApiRequestOptions, getApiUrl, getRawRequestOptions, getRawUrl, - GithubReaderProcessor, + GithubUrlReader, ProviderConfig, readConfig, -} from './GithubReaderProcessor'; +} from './GithubUrlReader'; -describe('GithubReaderProcessor', () => { +describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { it('sets the correct API version', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect((getApiRequestOptions(config).headers as any).Accept).toEqual( 'application/vnd.github.v3.raw', ); @@ -38,12 +36,12 @@ describe('GithubReaderProcessor', () => { it('inserts a token when needed', () => { const withToken: ProviderConfig = { - target: '', + host: '', apiBaseUrl: '', token: 'A', }; const withoutToken: ProviderConfig = { - target: '', + host: '', apiBaseUrl: '', }; expect( @@ -58,12 +56,12 @@ describe('GithubReaderProcessor', () => { describe('getRawRequestOptions', () => { it('inserts a token when needed', () => { const withToken: ProviderConfig = { - target: '', + host: '', rawBaseUrl: '', token: 'A', }; const withoutToken: ProviderConfig = { - target: '', + host: '', rawBaseUrl: '', }; expect( @@ -77,13 +75,13 @@ describe('GithubReaderProcessor', () => { describe('getApiUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { const config: ProviderConfig = { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', }; expect( @@ -110,7 +108,7 @@ describe('GithubReaderProcessor', () => { it('happy path for ghe', () => { const config: ProviderConfig = { - target: 'https://ghe.mycompany.net', + host: 'ghe.mycompany.net', apiBaseUrl: 'https://ghe.mycompany.net/api/v3', }; expect( @@ -128,13 +126,13 @@ describe('GithubReaderProcessor', () => { describe('getRawUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { const config: ProviderConfig = { - target: 'https://github.com', + host: 'github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }; expect( @@ -151,7 +149,7 @@ describe('GithubReaderProcessor', () => { it('happy path for ghe', () => { const config: ProviderConfig = { - target: 'https://ghe.mycompany.net', + host: 'ghe.mycompany.net', rawBaseUrl: 'https://ghe.mycompany.net/raw', }; expect( @@ -167,23 +165,23 @@ describe('GithubReaderProcessor', () => { describe('readConfig', () => { function config( - providers: { target: string; apiBaseUrl?: string; token?: string }[], + providers: { host: string; apiBaseUrl?: string; token?: string }[], ) { return ConfigReader.fromConfigs([ { context: '', data: { - catalog: { processors: { github: { providers } } }, + integrations: { github: providers }, }, }, ]); } it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([]), getVoidLogger()); + const output = readConfig(config([])); expect(output).toEqual([ { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }, @@ -191,13 +189,10 @@ describe('GithubReaderProcessor', () => { }); it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig( - config([{ target: 'https://github.com' }]), - getVoidLogger(), - ); + const output = readConfig(config([{ host: 'github.com' }])); expect(output).toEqual([ { - target: 'https://github.com', + host: 'github.com', apiBaseUrl: 'https://api.github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }, @@ -205,64 +200,33 @@ describe('GithubReaderProcessor', () => { }); it('rejects custom targets with no base URLs', () => { - expect(() => - readConfig( - config([{ target: 'https://ghe.company.com' }]), - getVoidLogger(), - ), - ).toThrow( - 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', + expect(() => readConfig(config([{ host: 'ghe.company.com' }]))).toThrow( + "GitHub integration for 'ghe.company.com' must configure an explicit apiBaseUrl and rawBaseUrl", ); }); it('rejects funky configs', () => { + expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/); + expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/); expect(() => - readConfig(config([{ target: 7 } as any]), getVoidLogger()), - ).toThrow(/target/); - expect(() => - readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()), - ).toThrow(/target/); - expect(() => - readConfig( - config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]), - getVoidLogger(), - ), + readConfig(config([{ host: 'github.com', apiBaseUrl: 7 } as any])), ).toThrow(/apiBaseUrl/); expect(() => - readConfig( - config([{ target: 'https://github.com', token: 7 } as any]), - getVoidLogger(), - ), + readConfig(config([{ host: 'github.com', token: 7 } as any])), ).toThrow(/token/); }); }); describe('implementation', () => { - it('rejects unknown types', async () => { - const processor = new GithubReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'not-github', - target: 'https://github.com', - }; - await expect( - processor.readLocation(location, false, () => {}), - ).resolves.toBeFalsy(); - }); - it('rejects unknown targets', async () => { - const processor = new GithubReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); - const location: LocationSpec = { - type: 'github', - target: 'https://not.github.com/apa', - }; + const processor = new GithubUrlReader({ + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }); await expect( - processor.readLocation(location, false, () => {}), + processor.read('https://not.github.com/apa'), ).rejects.toThrow( - /There is no GitHub provider that matches https:\/\/not.github.com\/apa/, + 'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path', ); }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts new file mode 100644 index 0000000000..e5bed6dd26 --- /dev/null +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -0,0 +1,236 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch'; +import { NotFoundError } from '../errors'; +import { ReaderFactory, UrlReader } from './types'; + +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The host of the target that this matches on, e.g. "github.com" + */ + host: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Invalid GitHub URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Invalid GitHub URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + const providerConfigs = + config.getOptionalConfigArray('integrations.github') ?? []; + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'github.com'; + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === 'github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (host === 'github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `GitHub integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`, + ); + } + + providers.push({ host, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.host === 'github.com')) { + providers.push({ + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubUrlReader implements UrlReader { + private config: ProviderConfig; + + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(provider => { + const reader = new GithubUrlReader(provider); + const predicate = (url: URL) => url.host === provider.host; + return { reader, predicate }; + }); + }; + + constructor(config: ProviderConfig) { + this.config = config; + } + + async read(url: string): Promise { + const useApi = + this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl); + const ghUrl = useApi + ? getApiUrl(url, this.config) + : getRawUrl(url, this.config); + const options = useApi + ? getApiRequestOptions(this.config) + : getRawRequestOptions(this.config); + + let response: Response; + try { + response = await fetch(ghUrl.toString(), options); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.ok) { + return response.buffer(); + } + + const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + toString() { + const { host, token } = this.config; + return `github{host=${host},authed=${Boolean(token)}}`; + } +} diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts new file mode 100644 index 0000000000..09da9c4e0a --- /dev/null +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { GitlabUrlReader } from './GitlabUrlReader'; + +const logger = getVoidLogger(); + +describe('GitlabUrlReader', () => { + const worker = setupServer(); + + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), + ), + ); + }); + afterEach(() => worker.resetHandlers()); + + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { gitlab: [{ host: 'gitlab.com', token }] }, + }, + 'test-config', + ); + + it.each([ + // Project URLs + { + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '', + }), + }), + }, + { + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + url: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + headers: expect.objectContaining({ + 'private-token': '0123456789', + }), + }), + }, + { + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }), + }, + + // Raw URLs + { + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { + const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + await reader.read(url); + }).rejects.toThrow(error); + }); +}); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts new file mode 100644 index 0000000000..0e430d650c --- /dev/null +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -0,0 +1,197 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch, { RequestInit, Response } from 'node-fetch'; +import { Config } from '@backstage/config'; +import { NotFoundError } from '../errors'; +import { ReaderFactory, UrlReader } from './types'; + +type Options = { + host: string; + token?: string; +}; + +function readConfig(config: Config): Options[] { + const optionsArr = Array(); + + const providerConfigs = + config.getOptionalConfigArray('integrations.gitlab') ?? []; + + for (const providerConfig of providerConfigs) { + const host = providerConfig.getOptionalString('host') ?? 'gitlab.com'; + const token = providerConfig.getOptionalString('token'); + + optionsArr.push({ host, token }); + } + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public gitlab repos. + if (!optionsArr.some(p => p.host === 'gitlab.com')) { + optionsArr.push({ host: 'gitlab.com' }); + } + + return optionsArr; +} + +export class GitlabUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + return readConfig(config).map(options => { + const reader = new GitlabUrlReader(options); + const predicate = (url: URL) => url.host === options.host; + return { reader, predicate }; + }); + }; + + constructor(private readonly options: Options) {} + + async read(url: string): Promise { + // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used + // the existence of /~/blob/ to switch the logic. Don't know if this + // makes sense and it might require some more work. + let builtUrl: URL; + if (url.includes('/-/blob/')) { + const projectID = await this.getProjectID(url); + builtUrl = this.buildProjectUrl(url, projectID); + } else { + builtUrl = this.buildRawUrl(url); + } + + let response: Response; + try { + response = await fetch(builtUrl.toString(), this.getRequestOptions()); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.ok) { + return response.buffer(); + } + + const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + // Converts + // from: https://gitlab.example.com/a/b/blob/master/c.yaml + // to: https://gitlab.example.com/a/b/raw/master/c.yaml + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitLab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( + '/', + ); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch + private buildProjectUrl(target: string, projectID: Number): URL { + try { + const url = new URL(target); + + const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + + const [branch, ...filePath] = branchAndfilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + private async getProjectID(target: string): Promise { + const url = new URL(target); + + if ( + // absPaths to gitlab files should contain /-/blob + // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + !url.pathname.match(/\/\-\/blob\//) + ) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Find ProjectID from url + // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' + // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + this.getRequestOptions(), + ); + const projectIDJson = await response.json(); + const projectID: Number = projectIDJson.id; + + return projectID; + } catch (e) { + throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); + } + } + + private getRequestOptions(): RequestInit { + return { + headers: { + ['PRIVATE-TOKEN']: this.options.token ?? '', + }, + }; + } + + toString() { + const { host, token } = this.options; + return `gitlab{host=${host},authed=${Boolean(token)}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts new file mode 100644 index 0000000000..7654cc8aac --- /dev/null +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader, UrlReaderPredicateTuple } from './types'; + +type Options = { + // UrlReader to fall back to if no other reader is matched + fallback?: UrlReader; +}; + +/** + * A UrlReader implementation that selects from a set of UrlReaders + * based on a predicate tied to each reader. + */ +export class UrlReaderPredicateMux implements UrlReader { + private readonly readers: UrlReaderPredicateTuple[] = []; + private readonly fallback?: UrlReader; + + constructor({ fallback }: Options) { + this.fallback = fallback; + } + + register(tuple: UrlReaderPredicateTuple): void { + this.readers.push(tuple); + } + + read(url: string): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + return reader.read(url); + } + } + + if (this.fallback) { + return this.fallback.read(url); + } + + throw new Error(`No reader found that could handle '${url}'`); + } + + toString() { + return `predicateMux{readers=${this.readers + .map(t => t.reader) + .join(',')},fallback=${this.fallback}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts new file mode 100644 index 0000000000..e1d99a2c49 --- /dev/null +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; +import { ReaderFactory, UrlReader } from './types'; +import { UrlReaderPredicateMux } from './UrlReaderPredicateMux'; +import { AzureUrlReader } from './AzureUrlReader'; +import { BitbucketUrlReader } from './BitbucketUrlReader'; +import { GithubUrlReader } from './GithubUrlReader'; +import { GitlabUrlReader } from './GitlabUrlReader'; +import { FetchUrlReader } from './FetchUrlReader'; + +type CreateOptions = { + /** Root config object */ + config: Config; + /** Logger used by all the readers */ + logger: Logger; + /** A list of factories used to construct individual readers that match on URLs */ + factories?: ReaderFactory[]; + /** Fallback reader to use if none of the readers created by the factories match */ + fallback?: UrlReader; +}; + +/** + * UrlReaders provide various utilities related to the UrlReader interface. + */ +export class UrlReaders { + /** + * Creates a UrlReader without any known types. + */ + static create({ + logger, + config, + factories, + fallback, + }: CreateOptions): UrlReader { + const mux = new UrlReaderPredicateMux({ fallback: fallback }); + + for (const factory of factories ?? []) { + const tuples = factory({ config, logger: logger }); + + for (const tuple of tuples) { + mux.register(tuple); + } + } + + return mux; + } + + /** + * Creates a UrlReader that includes all the default factories from this package. + * + * Any additional factories passed will be loaded before the default ones. + * + * If no fallback reader is passed, a plain fetch reader will be used. + */ + static default({ logger, config, factories = [], fallback }: CreateOptions) { + return UrlReaders.create({ + logger, + config, + factories: factories.concat([ + AzureUrlReader.factory, + BitbucketUrlReader.factory, + GithubUrlReader.factory, + GitlabUrlReader.factory, + ]), + fallback: fallback ?? new FetchUrlReader(), + }); + } +} diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts new file mode 100644 index 0000000000..8ecc08ebca --- /dev/null +++ b/packages/backend-common/src/reading/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { UrlReader } from './types'; +export { UrlReaders } from './UrlReaders'; +export { AzureUrlReader } from './AzureUrlReader'; +export { BitbucketUrlReader } from './BitbucketUrlReader'; +export { GithubUrlReader } from './GithubUrlReader'; +export { GitlabUrlReader } from './GitlabUrlReader'; diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts new file mode 100644 index 0000000000..e423db3eca --- /dev/null +++ b/packages/backend-common/src/reading/types.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +/** + * A generic interface for fetching plain data from URLs. + */ +export type UrlReader = { + read(url: string): Promise; +}; + +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + +/** + * A factory function that can read config to construct zero or more + * UrlReaders along with a predicate for when it should be used. + */ +export type ReaderFactory = (options: { + config: Config; + logger: Logger; +}) => UrlReaderPredicateTuple[]; diff --git a/packages/backend-common/src/setupTests.ts b/packages/backend-common/src/setupTests.ts index f7b6ca962d..ba33cf996b 100644 --- a/packages/backend-common/src/setupTests.ts +++ b/packages/backend-common/src/setupTests.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -require('jest-fetch-mock').enableMocks(); - export {}; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index e86034dd29..959af71427 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -32,6 +32,7 @@ import { useHotMemoize, notFoundHandler, SingleHostDiscovery, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; @@ -49,9 +50,14 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); + const root = getRootLogger(); + const reader = UrlReaders.default({ logger: root, config }); + const discovery = SingleHostDiscovery.fromConfig(config); + + root.info(`Created UrlReader ${reader}`); return (plugin: string): PluginEnvironment => { - const logger = getRootLogger().child({ type: 'plugin', plugin }); + const logger = root.child({ type: 'plugin', plugin }); const database = createDatabaseClient( config.getConfig('backend.database'), { @@ -60,8 +66,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); - const discovery = SingleHostDiscovery.fromConfig(config); - return { logger, database, config, discovery }; + return { logger, database, config, reader, discovery }; }; } diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 219b04a0f7..c1d390f8cc 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -28,10 +28,11 @@ import { useHotCleanup } from '@backstage/backend-common'; export default async function createPlugin({ logger, - database, config, + reader, + database, }: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, reader, config }); const db = await DatabaseManager.createDatabase(database, { logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 3709fc8d9a..9257fcc9cf 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -17,11 +17,12 @@ import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; database: Knex; config: Config; + reader: UrlReader; discovery: PluginEndpointDiscovery; }; diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index f1278bd3c3..a179efe982 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -82,31 +82,31 @@ catalog: # env: GHE_PRIVATE_TOKEN locations: # Backstage example components - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml # Backstage example APIs - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml # Backstage example templates - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml rules: - allow: [Template] - - type: github + - type: url target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml rules: - allow: [Template] diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 47ab2c5993..684db8aab8 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -16,6 +16,7 @@ import { useHotMemoize, notFoundHandler, SingleHostDiscovery, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; import auth from './plugins/auth'; @@ -27,9 +28,14 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); + const root = getRootLogger(); + const reader = UrlReaders.default({ logger: root, config }); + const discovery = SingleHostDiscovery.fromConfig(config); + + root.info(`Created UrlReader ${reader}`); return (plugin: string): PluginEnvironment => { - const logger = getRootLogger().child({ type: 'plugin', plugin }); + const logger = root.child({ type: 'plugin', plugin }); const database = createDatabaseClient( config.getConfig('backend.database'), { @@ -38,8 +44,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { }, }, ); - const discovery = SingleHostDiscovery.fromConfig(config); - return { logger, database, config, discovery }; + return { logger, database, config, reader, discovery }; }; } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts index aa0538a6d8..345ac4c23d 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -13,9 +13,10 @@ import { useHotCleanup } from '@backstage/backend-common'; export default async function createPlugin({ logger, config, + reader, database, }: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, reader, config }); const db = await DatabaseManager.createDatabase(database, { logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); diff --git a/packages/create-app/templates/default-app/packages/backend/src/types.ts b/packages/create-app/templates/default-app/packages/backend/src/types.ts index d145255390..aa003f63a5 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/types.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/types.ts @@ -1,11 +1,12 @@ import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; database: Knex; config: Config; + reader: UrlReader discovery: PluginEndpointDiscovery; }; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index e964524bc7..e82699f526 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -74,11 +74,11 @@ export class CatalogRulesEnforcer { * - allow: [Component, API] * * locations: - * - type: github + * - type: url * target: https://github.com/org/repo/blob/master/users.yaml * rules: * - allow: [User, Group] - * - type: github + * - type: url * target: https://github.com/org/repo/blob/master/systems.yaml * rules: * - allow: [System] diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 00ee95f3a5..96ffaf398c 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; import { Entity, EntityPolicies, @@ -57,6 +57,7 @@ import { LocationReader, ReadLocationResult } from './types'; const MAX_DEPTH = 10; type Options = { + reader: UrlReader; logger?: Logger; config?: Config; processors?: LocationProcessor[]; @@ -72,6 +73,7 @@ export class LocationReaders implements LocationReader { static defaultProcessors(options: { logger: Logger; + reader: UrlReader; config?: Config; entityPolicy?: EntityPolicy; }): LocationProcessor[] { @@ -80,16 +82,46 @@ export class LocationReaders implements LocationReader { config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; + + // TODO(Rugvip): These are added for backwards compatibility if config exists + // The idea is to have everyone migrate from using the old processors to the new + // integration config driven UrlReaders. In an upcoming release we can then completely + // remove support for the old processors, but still keep handling the deprecated location + // types for a while, but with a warning. + const oldProcessors = []; + const pc = config.getOptionalConfig('catalog.processors'); + if (pc?.has('github')) { + logger.warn( + `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, + ); + oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); + } + if (pc?.has('gitlabApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, + ); + oldProcessors.push(new GitlabApiReaderProcessor(config)); + oldProcessors.push(new GitlabReaderProcessor()); + } + if (pc?.has('bitbucketApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, + ); + oldProcessors.push(new BitbucketApiReaderProcessor(config)); + } + if (pc?.has('azureApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, + ); + oldProcessors.push(new AzureApiReaderProcessor(config)); + } + return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - GithubReaderProcessor.fromConfig(config, logger), - new GitlabApiReaderProcessor(config), - new GitlabReaderProcessor(), - new BitbucketApiReaderProcessor(config), - new AzureApiReaderProcessor(config), + ...oldProcessors, GithubOrgReaderProcessor.fromConfig(config), - new UrlReaderProcessor(), + new UrlReaderProcessor(options), new YamlProcessor(), PlaceholderProcessor.default(), new CodeOwnersProcessor(), @@ -103,7 +135,8 @@ export class LocationReaders implements LocationReader { constructor({ logger = getVoidLogger(), config, - processors = LocationReaders.defaultProcessors({ logger, config }), + reader, + processors = LocationReaders.defaultProcessors({ logger, reader, config }), }: Options) { this.logger = logger; this.processors = processors; diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts deleted file mode 100644 index 9b234e3e75..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; -import { ConfigReader } from '@backstage/config'; - -describe('AzureApiReaderProcessor', () => { - const createConfig = (token: string | undefined) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { - processors: { - azureApi: { - privateToken: token, - }, - }, - }, - }, - }, - ]); - - it('should build raw api', () => { - const processor = new AzureApiReaderProcessor(createConfig(undefined)); - const tests = [ - { - target: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', - url: new URL( - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', - ), - err: undefined, - }, - { - target: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', - url: new URL( - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - ]; - - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } - }); - - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - Authorization: 'Basic OjAxMjM0NTY3ODk=', - }, - }, - }, - { - token: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string", - }, - { - token: undefined, - expect: { - headers: {}, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => new AzureApiReaderProcessor(createConfig(test.token)), - ).toThrowError(test.err); - } else { - const processor = new AzureApiReaderProcessor(createConfig(test.token)); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts index 70a84d72df..2a15c18c84 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -20,6 +20,11 @@ import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; import { Config } from '@backstage/config'; +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + export class AzureApiReaderProcessor implements LocationProcessor { private privateToken: string; diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts deleted file mode 100644 index 953f0703be..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; -import { ConfigReader } from '@backstage/config'; - -describe('BitbucketApiReaderProcessor', () => { - const createConfig = ( - username: string | undefined, - appPassword: string | undefined, - ) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { - processors: { - bitbucketApi: { - username: username, - appPassword: appPassword, - }, - }, - }, - }, - }, - ]); - - it('should build raw api', () => { - const processor = new BitbucketApiReaderProcessor( - createConfig(undefined, undefined), - ); - - const tests = [ - { - target: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - url: new URL( - 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', - ), - err: undefined, - }, - { - target: 'https://api.com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', - }, - { - target: 'com/a/b/blob/master/path/to/c.yaml', - url: null, - err: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - ]; - - for (const test of tests) { - if (test.err) { - expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); - } else if (test.url) { - expect(processor.buildRawUrl(test.target).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } - }); - - it('should return request options', () => { - const tests = [ - { - username: '', - password: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", - }, - { - username: 'only-user-provided', - password: '', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string", - }, - { - username: '', - password: 'only-password-provided', - expect: { - headers: {}, - }, - err: - "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", - }, - { - username: 'some-user', - password: 'my-secret', - expect: { - headers: { - Authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', - }, - }, - }, - { - username: undefined, - password: undefined, - expect: { - headers: {}, - }, - }, - { - username: 'only-user-provided', - password: undefined, - expect: { - headers: {}, - }, - }, - { - username: undefined, - password: 'only-password-provided', - expect: { - headers: {}, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => - new BitbucketApiReaderProcessor( - createConfig(test.username, test.password), - ), - ).toThrowError(test.err); - } else { - const processor = new BitbucketApiReaderProcessor( - createConfig(test.username, test.password), - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts index c46fbd1737..b3720a5b76 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -20,6 +20,11 @@ import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; import { Config } from '@backstage/config'; +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + export class BitbucketApiReaderProcessor implements LocationProcessor { private username: string; private password: string; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index cc50e53240..89d1f69860 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -22,6 +22,11 @@ import { Logger } from 'winston'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + /** * The configuration parameters for a single GitHub API provider. */ diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts deleted file mode 100644 index 7e55fece1b..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; -import { ConfigReader } from '@backstage/config'; - -describe('GitlabApiReaderProcessor', () => { - const createConfig = (token: string | undefined) => - ConfigReader.fromConfigs([ - { - context: '', - data: { - catalog: { - processors: { - gitlabApi: { - privateToken: token, - }, - }, - }, - }, - }, - ]); - - it('should build raw api', () => { - const processor = new GitlabApiReaderProcessor(createConfig(undefined)); - - const tests = [ - { - target: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - url: new URL( - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - { - target: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', - url: new URL( - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - { - target: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup - url: new URL( - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', - ), - err: undefined, - }, - ]; - - for (const test of tests) { - if (test.url) { - expect(processor.buildRawUrl(test.target, 12345).toString()).toEqual( - test.url.toString(), - ); - } else { - throw new Error( - 'This should not have happened. Either err or url should have matched.', - ); - } - } - }); - - it('should return request options', () => { - const tests = [ - { - token: '0123456789', - expect: { - headers: { - 'PRIVATE-TOKEN': '0123456789', - }, - }, - }, - { - token: '', - err: - "Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string", - expect: { - headers: { - 'PRIVATE-TOKEN': '', - }, - }, - }, - { - token: undefined, - expect: { - headers: { - 'PRIVATE-TOKEN': '', - }, - }, - }, - ]; - - for (const test of tests) { - if (test.err) { - expect( - () => new GitlabApiReaderProcessor(createConfig(test.token)), - ).toThrowError(test.err); - } else { - const processor = new GitlabApiReaderProcessor( - createConfig(test.token), - ); - expect(processor.getRequestOptions()).toEqual(test.expect); - } - } - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts index a51c306777..aeba7a7ebe 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -20,6 +20,11 @@ import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; import { Config } from '@backstage/config'; +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + export class GitlabApiReaderProcessor implements LocationProcessor { private privateToken: string; diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts index c33b388e1a..b51bf85a8e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -19,6 +19,11 @@ import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + export class GitlabReaderProcessor implements LocationProcessor { async readLocation( location: LocationSpec, diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index 1ab4cb09b8..009872cda3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -22,6 +22,8 @@ import { } from './types'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost:23000'; @@ -32,7 +34,9 @@ describe('UrlReaderProcessor', () => { afterAll(() => server.close()); it('should load from url', async () => { - const processor = new UrlReaderProcessor(); + const logger = getVoidLogger(); + const reader = UrlReaders.default({ logger, config: new ConfigReader({}) }); + const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', target: `${mockApiOrigin}/component.yaml`, @@ -54,7 +58,9 @@ describe('UrlReaderProcessor', () => { }); it('should fail load from url with error', async () => { - const processor = new UrlReaderProcessor(); + const logger = getVoidLogger(); + const reader = UrlReaders.default({ logger, config: new ConfigReader({}) }); + const processor = new UrlReaderProcessor({ reader, logger }); const spec = { type: 'url', target: `${mockApiOrigin}/component-notfound.yaml`, @@ -74,7 +80,7 @@ describe('UrlReaderProcessor', () => { expect(generated.location).toBe(spec); expect(generated.error.name).toBe('NotFoundError'); expect(generated.error.message).toBe( - `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`, + `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`, ); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 0c879ea70c..2bc05bf1cb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -14,40 +14,58 @@ * limitations under the License. */ +import { Logger } from 'winston'; +import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +// TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this +// can be removed in a bit +const deprecatedTypes = [ + 'github', + 'github/api', + 'bitbucket/api', + 'gitlab/api', + 'azure/api', +]; + +type Options = { + reader: UrlReader; + logger: Logger; +}; + export class UrlReaderProcessor implements LocationProcessor { + constructor(private readonly options: Options) {} + async readLocation( location: LocationSpec, optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'url') { + if (deprecatedTypes.includes(location.type)) { + // TODO(Rugvip): Let's not enable this warning yet, as we want to move over the example YAMLs + // in this repo to use the 'url' type first. + // this.options.logger.warn( + // `Using deprecated location type '${location.type}' for '${location.target}', use 'url' instead`, + // ); + } else if (location.type !== 'url') { return false; } try { - const response = await fetch(location.target); + const data = await this.options.reader.read(location.target); + emit(result.data(location, data)); + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); } + } else { + emit(result.generalError(location, message)); } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); } return true; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 0b532d1664..5939c7eb58 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -17,6 +17,7 @@ import { createServiceBuilder, loadBackendConfig, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { Server } from 'http'; @@ -39,12 +40,13 @@ export async function startStandaloneServer( ): Promise { const logger = options.logger.child({ service: 'catalog-backend' }); const config = ConfigReader.fromConfigs(await loadBackendConfig()); + const reader = UrlReaders.default({ logger, config }); logger.debug('Creating application...'); const db = await DatabaseManager.createInMemoryDatabase({ logger }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); - const locationReader = new LocationReaders({ logger, config }); + const locationReader = new LocationReaders({ logger, config, reader }); const higherOrderOperation = new HigherOrderOperations( entitiesCatalog, locationsCatalog,