From 36a71d2780b2d355936542409eec1bc3a4a2a558 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 26 Oct 2020 15:46:22 +0100 Subject: [PATCH] catalog-backend: removed deprecated location processors --- .changeset/polite-laws-think.md | 5 + .../templates/default-app/app-config.yaml.hbs | 22 +- .../processors/AzureApiReaderProcessor.ts | 146 ---------- .../processors/BitbucketApiReaderProcessor.ts | 138 --------- .../processors/GithubReaderProcessor.ts | 274 ------------------ .../processors/GitlabApiReaderProcessor.ts | 141 --------- .../processors/GitlabReaderProcessor.ts | 94 ------ .../src/ingestion/processors/index.ts | 5 - .../src/service/CatalogBuilder.ts | 38 +-- 9 files changed, 26 insertions(+), 837 deletions(-) create mode 100644 .changeset/polite-laws-think.md delete mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts diff --git a/.changeset/polite-laws-think.md b/.changeset/polite-laws-think.md new file mode 100644 index 0000000000..a5eb7f9883 --- /dev/null +++ b/.changeset/polite-laws-think.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Removed support for deprecated `catalog.providers` config that have been moved to `integrations` diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index a0b9f156ed..1f265d7e7d 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -39,6 +39,17 @@ backend: # $file: /ca/server.crt {{/if}} +integrations: + github: + - host: github.com + token: + $env: GITHUB_TOKEN + ### Example for how to add your GitHub Enterprise instance using the API: + # - host: ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $env: GHE_TOKEN + proxy: '/test': target: 'https://example.com' @@ -66,17 +77,6 @@ scaffolder: catalog: rules: - allow: [Component, API, Group, User, Template, Location] - processors: - github: - providers: - - target: https://github.com - token: - $env: GITHUB_TOKEN - # Example for how to add your GitHub Enterprise instance: - # - target: https://ghe.example.net - # apiBaseUrl: https://ghe.example.net/api/v3 - # token: - # $env: GHE_TOKEN locations: # Backstage example components - type: url diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts deleted file mode 100644 index 9b9ba586ef..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; - -import { Config } from '@backstage/config'; -import fetch from 'cross-fetch'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// *********************************************************************** -// * NOTE: This has been replaced by packages/backend-common/src/reading * -// * Don't implement new functionality here as this file will be removed * -// *********************************************************************** - -export class AzureApiReaderProcessor implements CatalogProcessor { - private privateToken: string; - - constructor(config: Config) { - this.privateToken = - config.getOptionalString('catalog.processors.azureApi.privateToken') ?? - ''; - } - - getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; - - if (this.privateToken !== '') { - headers.Authorization = `Basic ${Buffer.from( - `:${this.privateToken}`, - 'utf8', - ).toString('base64')}`; - } - - return { - headers, - }; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - if (location.type !== 'azure/api') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html - if (response.ok && response.status !== 203) { - const data = Buffer.from(await response.text()); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - return true; - } - - // Converts - // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents - // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} - buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - project, - srcKeyword, - repoName, - ] = url.pathname.split('/'); - - const path = url.searchParams.get('path') || ''; - const ref = url.searchParams.get('version')?.substr(2); - - if ( - url.hostname !== 'dev.azure.com' || - empty !== '' || - userOrOrg === '' || - project === '' || - srcKeyword !== '_git' || - repoName === '' || - path === '' || - ref === '' - ) { - throw new Error('Wrong Azure Devops URL or Invalid file path'); - } - - // transform to api - url.pathname = [ - empty, - userOrOrg, - project, - '_apis', - 'git', - 'repositories', - repoName, - 'items', - ].join('/'); - - const queryParams = [`path=${path}`]; - - if (ref) { - queryParams.push(`version=${ref}`); - } - - url.search = queryParams.join('&'); - - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts deleted file mode 100644 index 337053965f..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; -import * as result from './results'; -import { Config } from '@backstage/config'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// *********************************************************************** -// * NOTE: This has been replaced by packages/backend-common/src/reading * -// * Don't implement new functionality here as this file will be removed * -// *********************************************************************** - -export class BitbucketApiReaderProcessor implements CatalogProcessor { - private username: string; - private password: string; - - constructor(config: Config) { - this.username = - config.getOptionalString('catalog.processors.bitbucketApi.username') ?? - ''; - this.password = - config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? - ''; - } - - getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; - - if (this.username !== '' && this.password !== '') { - headers.Authorization = `Basic ${Buffer.from( - `${this.username}:${this.password}`, - 'utf8', - ).toString('base64')}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - if (location.type !== 'bitbucket/api') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - if (response.ok) { - const data = await response.text(); - emit(result.data(location, Buffer.from(data))); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - return true; - } - - // Converts - // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml - // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml - - buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - srcKeyword, - ref, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'bitbucket.org' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - srcKeyword !== 'src' - ) { - throw new Error('Wrong Bitbucket URL or Invalid file path'); - } - - // transform to api - url.pathname = [ - empty, - '2.0', - 'repositories', - userOrOrg, - repoName, - 'src', - ref, - ...restOfPath, - ].join('/'); - url.hostname = 'api.bitbucket.org'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts deleted file mode 100644 index fed7f2a5c2..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import parseGitUri from 'git-url-parse'; -import fetch from 'cross-fetch'; -import { Logger } from 'winston'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// *********************************************************************** -// * NOTE: This has been replaced by packages/backend-common/src/reading * -// * Don't implement new functionality here as this file will be removed * -// *********************************************************************** - -/** - * The configuration parameters for a single GitHub API provider. - */ -export type ProviderConfig = { - /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. - */ - target: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.github.com", - * with no trailing slash. - * - * May be omitted specifically for GitHub; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - apiBaseUrl?: string; - - /** - * The base URL of the raw fetch endpoint of this provider, e.g. - * "https://raw.githubusercontent.com", with no trailing slash. - * - * May be omitted specifically for GitHub; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - rawBaseUrl?: string; - - /** - * The authorization token to use for requests to this provider. - * - * If no token is specified, anonymous access is used. - */ - token?: string; -}; - -export function getApiRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -export function getRawRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = {}; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/path/to/c.yaml -// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getApiUrl(target: string, provider: ProviderConfig): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - - if ( - !owner || - !name || - !ref || - (filepathtype !== 'blob' && filepathtype !== 'raw') - ) { - throw new Error('Wrong URL or invalid file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - return new URL( - `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/c.yaml -// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml -export function getRawUrl(target: string, provider: ProviderConfig): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - - if ( - !owner || - !name || - !ref || - (filepathtype !== 'blob' && filepathtype !== 'raw') - ) { - throw new Error('Wrong URL or invalid file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - return new URL( - `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -export function readConfig(config: Config, logger: Logger): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - // TODO(freben): Deprecate the old config root entirely in a later release - if (config.has('catalog.processors.githubApi')) { - logger.warn( - 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', - ); - } - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. - const providerConfigs = - config.getOptionalConfigArray('catalog.processors.github.providers') ?? - config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? - []; - const legacyToken = - config.getOptionalString('catalog.processors.github.privateToken') ?? - config.getOptionalString('catalog.processors.githubApi.privateToken'); - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); - const token = providerConfig.getOptionalString('token'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { - apiBaseUrl = 'https://api.github.com'; - } - - if (rawBaseUrl) { - rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { - rawBaseUrl = 'https://raw.githubusercontent.com'; - } - - if (!apiBaseUrl && !rawBaseUrl) { - throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, - ); - } - - providers.push({ target, apiBaseUrl, rawBaseUrl, token }); - } - - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { - providers.push({ - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - rawBaseUrl: 'https://raw.githubusercontent.com', - token: legacyToken, - }); - } - - return providers; -} - -/** - * A processor that adds the ability to read files from GitHub v3 APIs, such as - * the one exposed by GitHub itself. - */ -export class GithubReaderProcessor implements CatalogProcessor { - private providers: ProviderConfig[]; - - static fromConfig(config: Config, logger: Logger) { - return new GithubReaderProcessor(readConfig(config, logger)); - } - - constructor(providers: ProviderConfig[]) { - this.providers = providers; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - // The github/api type is for backward compatibility - if (location.type !== 'github' && location.type !== 'github/api') { - return false; - } - - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, - ); - } - - try { - const useApi = - provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); - const url = useApi - ? getApiUrl(location.target, provider) - : getRawUrl(location.target, provider); - const options = useApi - ? getApiRequestOptions(provider) - : getRawRequestOptions(provider); - const response = await fetch(url.toString(), options); - - if (response.ok) { - const data = await response.text(); - emit(result.data(location, Buffer.from(data))); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts deleted file mode 100644 index 51a286c7ea..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; -import { Config } from '@backstage/config'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// *********************************************************************** -// * NOTE: This has been replaced by packages/backend-common/src/reading * -// * Don't implement new functionality here as this file will be removed * -// *********************************************************************** - -export class GitlabApiReaderProcessor implements CatalogProcessor { - private privateToken: string; - - constructor(config: Config) { - this.privateToken = - config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? - ''; - } - - getRequestOptions(): RequestInit { - const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; - if (this.privateToken !== '') { - headers['PRIVATE-TOKEN'] = this.privateToken; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - if (location.type !== 'gitlab/api') { - return false; - } - - try { - const projectID = await this.getProjectID(location.target); - const url = this.buildRawUrl(location.target, projectID); - const response = await fetch(url.toString(), this.getRequestOptions()); - if (response.ok) { - const data = await response.text(); - emit(result.data(location, Buffer.from(data))); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - return true; - } - - // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch - buildRawUrl(target: string, projectID: Number): URL { - try { - const url = new URL(target); - - const branchAndfilePath = url.pathname.split('/-/blob/')[1]; - - const [branch, ...filePath] = branchAndfilePath.split('/'); - - url.pathname = [ - '/api/v4/projects', - projectID, - 'repository/files', - encodeURIComponent(filePath.join('/')), - 'raw', - ].join('/'); - url.search = `?ref=${branch}`; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } - - async getProjectID(target: string): Promise { - const url = new URL(target); - - if ( - // absPaths to gitlab files should contain /-/blob - // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - !url.pathname.match(/\/\-\/blob\//) - ) { - throw new Error('Please provide full path to yaml file from Gitlab'); - } - try { - const repo = url.pathname.split('/-/blob/')[0]; - - // Find ProjectID from url - // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' - // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' - const repoIDLookup = new URL( - `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( - repo.replace(/^\//, ''), - )}`, - ); - const response = await fetch( - repoIDLookup.toString(), - this.getRequestOptions(), - ); - const projectIDJson = await response.json(); - const projectID: Number = projectIDJson.id; - - return projectID; - } catch (e) { - throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts deleted file mode 100644 index 38498b44fa..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// *********************************************************************** -// * NOTE: This has been replaced by packages/backend-common/src/reading * -// * Don't implement new functionality here as this file will be removed * -// *********************************************************************** - -export class GitlabReaderProcessor implements CatalogProcessor { - async readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - if (location.type !== 'gitlab') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString()); - - if (response.ok) { - const data = await response.text(); - emit(result.data(location, Buffer.from(data))); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - throw result.notFoundError(location, message); - } - } else { - throw result.generalError(location, message); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } - - // Converts - // from: https://gitlab.example.com/a/b/blob/master/c.yaml - // to: https://gitlab.example.com/a/b/raw/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [empty, userOrOrg, repoName, , ...restOfPath] = url.pathname - .split('/') - // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml - .filter(path => path !== '-'); - - if ( - empty !== '' || - userOrOrg === '' || - repoName === '' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitLab URL'); - } - - // Replace 'blob' with 'raw' - url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( - '/', - ); - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 92303f04a3..8344678e5f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -20,14 +20,9 @@ export { results }; export * from './types'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; -export { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; -export { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -export { GithubReaderProcessor } from './GithubReaderProcessor'; -export { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; -export { GitlabReaderProcessor } from './GitlabReaderProcessor'; export { OwnerRelationProcessor } from './OwnerRelationProcessor'; export { LocationRefProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 4ec2a0834d..e795310915 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -43,15 +43,10 @@ import { import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, - AzureApiReaderProcessor, - BitbucketApiReaderProcessor, CatalogProcessor, CodeOwnersProcessor, FileReaderProcessor, GithubOrgReaderProcessor, - GithubReaderProcessor, - GitlabApiReaderProcessor, - GitlabReaderProcessor, OwnerRelationProcessor, HigherOrderOperation, HigherOrderOperations, @@ -318,6 +313,8 @@ export class CatalogBuilder { private buildProcessors(): CatalogProcessor[] { const { config, logger, reader } = this.env; + this.checkDeprecatedReaderProcessors(); + const placeholderResolvers: Record = { json: jsonPlaceholderResolver, yaml: yamlPlaceholderResolver, @@ -342,48 +339,33 @@ export class CatalogBuilder { return [ StaticLocationProcessor.fromConfig(config), new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), - ...this.buildDeprecatedReaderProcessors(), ...processors, ]; } - // TODO(Rugvip): These are added for backwards compatibility if config exists - // The idea is to have everyone migrate from using the old processors to - // the new integration config driven UrlReaders. In an upcoming release we - // can then completely remove support for the old processors, but still - // keep handling the deprecated location types for a while, but with a - // warning. - private buildDeprecatedReaderProcessors(): CatalogProcessor[] { - const { config, logger } = this.env; - - const result = []; - const pc = config.getOptionalConfig('catalog.processors'); + // TODO(Rugvip): These old processors are removed, for a while we'll be throwing + // errors here to make sure people know where to move the config + private checkDeprecatedReaderProcessors() { + const pc = this.env.config.getOptionalConfig('catalog.processors'); if (pc?.has('github')) { - logger.warn( + throw new Error( `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, ); - result.push(GithubReaderProcessor.fromConfig(config, logger)); } if (pc?.has('gitlabApi')) { - logger.warn( + throw new Error( `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, ); - result.push(new GitlabApiReaderProcessor(config)); - result.push(new GitlabReaderProcessor()); } if (pc?.has('bitbucketApi')) { - logger.warn( + throw new Error( `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, ); - result.push(new BitbucketApiReaderProcessor(config)); } if (pc?.has('azureApi')) { - logger.warn( + throw new Error( `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, ); - result.push(new AzureApiReaderProcessor(config)); } - - return result; } }