From 21159258a2218be241188ba74aaa93b237254d8f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Oct 2020 17:59:28 +0200 Subject: [PATCH] catalog-backend: added back readers for backwards-compatibility --- .../src/ingestion/LocationReaders.ts | 43 ++- .../processors/AzureApiReaderProcessor.ts | 147 ++++++++++ .../processors/BitbucketApiReaderProcessor.ts | 138 +++++++++ .../processors/GithubReaderProcessor.ts | 274 ++++++++++++++++++ .../processors/GitlabApiReaderProcessor.ts | 141 +++++++++ .../processors/GitlabReaderProcessor.ts | 94 ++++++ 6 files changed, 832 insertions(+), 5 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index d2a6b2404d..96ffaf398c 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,6 +32,7 @@ import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProc import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; +import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; @@ -77,16 +78,48 @@ export class LocationReaders implements LocationReader { entityPolicy?: EntityPolicy; }): LocationProcessor[] { const { + logger, config = new ConfigReader({}, 'missing-config'), entityPolicy = new EntityPolicies(), } = options; + + // TODO(Rugvip): These are added for backwards compatibility if config exists + // The idea is to have everyone migrate from using the old processors to the new + // integration config driven UrlReaders. In an upcoming release we can then completely + // remove support for the old processors, but still keep handling the deprecated location + // types for a while, but with a warning. + const oldProcessors = []; + const pc = config.getOptionalConfig('catalog.processors'); + if (pc?.has('github')) { + logger.warn( + `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, + ); + oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); + } + if (pc?.has('gitlabApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, + ); + oldProcessors.push(new GitlabApiReaderProcessor(config)); + oldProcessors.push(new GitlabReaderProcessor()); + } + if (pc?.has('bitbucketApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, + ); + oldProcessors.push(new BitbucketApiReaderProcessor(config)); + } + if (pc?.has('azureApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, + ); + oldProcessors.push(new AzureApiReaderProcessor(config)); + } + return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GitlabApiReaderProcessor(config), - new GitlabReaderProcessor(), - new BitbucketApiReaderProcessor(config), - new AzureApiReaderProcessor(config), + ...oldProcessors, GithubOrgReaderProcessor.fromConfig(config), new UrlReaderProcessor(options), new YamlProcessor(), @@ -96,7 +129,7 @@ export class LocationReaders implements LocationReader { new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), new AnnotateLocationEntityProcessor(), - ].flat(); + ]; } constructor({ diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..2a15c18c84 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.azureApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'items', + ].join('/'); + + const queryParams = [`path=${path}`]; + + if (ref) { + queryParams.push(`version=${ref}`); + } + + url.search = queryParams.join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts new file mode 100644 index 0000000000..b3720a5b76 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class BitbucketApiReaderProcessor implements LocationProcessor { + private username: string; + private password: string; + + constructor(config: Config) { + this.username = + config.getOptionalString('catalog.processors.bitbucketApi.username') ?? + ''; + this.password = + config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.username !== '' && this.password !== '') { + headers.Authorization = `Basic ${Buffer.from( + `${this.username}:${this.password}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'bitbucket/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + srcKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'bitbucket.org' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + srcKeyword !== 'src' + ) { + throw new Error('Wrong Bitbucket URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + '2.0', + 'repositories', + userOrOrg, + repoName, + 'src', + ref, + ...restOfPath, + ].join('/'); + url.hostname = 'api.bitbucket.org'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts new file mode 100644 index 0000000000..89d1f69860 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; +import { Logger } from 'winston'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config, logger: Logger): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // TODO(freben): Deprecate the old config root entirely in a later release + if (config.has('catalog.processors.githubApi')) { + logger.warn( + 'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead', + ); + } + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config, logger: Logger) { + return new GithubReaderProcessor(readConfig(config, logger)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } + + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + const response = await fetch(url.toString(), options); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts new file mode 100644 index 0000000000..aeba7a7ebe --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class GitlabApiReaderProcessor implements LocationProcessor { + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? + ''; + } + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; + if (this.privateToken !== '') { + headers['PRIVATE-TOKEN'] = this.privateToken; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab/api') { + return false; + } + + try { + const projectID = await this.getProjectID(location.target); + const url = this.buildRawUrl(location.target, projectID); + const response = await fetch(url.toString(), this.getRequestOptions()); + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch + buildRawUrl(target: string, projectID: Number): URL { + try { + const url = new URL(target); + + const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + + const [branch, ...filePath] = branchAndfilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + async getProjectID(target: string): Promise { + const url = new URL(target); + + if ( + // absPaths to gitlab files should contain /-/blob + // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + !url.pathname.match(/\/\-\/blob\//) + ) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Find ProjectID from url + // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' + // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + this.getRequestOptions(), + ); + const projectIDJson = await response.json(); + const projectID: Number = projectIDJson.id; + + return projectID; + } catch (e) { + throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts new file mode 100644 index 0000000000..b51bf85a8e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +// *********************************************************************** +// * NOTE: This has been replaced by packages/backend-common/src/reading * +// * Don't implement new functionality here as this file will be removed * +// *********************************************************************** + +export class GitlabReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + throw result.notFoundError(location, message); + } + } else { + throw result.generalError(location, message); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://gitlab.example.com/a/b/blob/master/c.yaml + // to: https://gitlab.example.com/a/b/raw/master/c.yaml + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [empty, userOrOrg, repoName, , ...restOfPath] = url.pathname + .split('/') + // for the common case https://gitlab.example.com/a/b/-/blob/master/c.yaml + .filter(path => path !== '-'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitLab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( + '/', + ); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +}