diff --git a/app-config.yaml b/app-config.yaml index 3349a6585f..bbc14e09f6 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -58,21 +58,23 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: $secret: env: GITHUB_PRIVATE_TOKEN - # Example for how to add your GitHub Enterprise instance: + #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net # apiBaseUrl: https://ghe.example.net/api/v3 # token: # $secret: # env: GHE_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional): + # - target: https://ghe.example.net + # rawBaseUrl: https://ghe.example.net/raw + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index ecac0613d9..d63176bb92 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -65,10 +65,6 @@ catalog: - allow: [Component, API, Group, Template, Location] processors: github: - privateToken: - $secret: - env: GITHUB_PRIVATE_TOKEN - githubApi: providers: - target: https://github.com token: diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index d1711cf3d5..bb62efdebb 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -27,6 +27,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", + "git-url-parse": "^11.2.0", "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", @@ -40,6 +41,7 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.21", + "@types/git-url-parse": "^9.0.0", "@types/lodash": "^4.14.151", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 6f3a27f00d..a564e9725c 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -27,7 +27,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; @@ -77,8 +76,7 @@ export class LocationReaders implements LocationReader { return [ StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), - new GithubReaderProcessor(config), - GithubApiReaderProcessor.fromConfig(config), + GithubReaderProcessor.fromConfig(config), new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts deleted file mode 100644 index 812bf6f488..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import fetch, { HeadersInit, RequestInit } from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -/** - * The configuration parameters for a single GitHub API provider. - */ -export type ProviderConfig = { - /** - * The prefix of the target that this matches on, e.g. "https://github.com", - * with no trailing slash. - */ - target: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.github.com", - * with no trailing slash. - */ - apiBaseUrl: string; - - /** - * The authorization token to use for requests to this provider. - * - * If no token is specified, anonymous API access is used. - */ - token?: string; -}; - -export function getRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/path/to/c.yaml -// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getRawUrl(target: string, provider: ProviderConfig): URL { - try { - const oldPath = new URL(target).pathname.split('/'); - const [, userOrOrg, repoName, blobOrRaw, ref, ...restOfPath] = oldPath; - - if ( - !userOrOrg || - !repoName || - (blobOrRaw !== 'blob' && blobOrRaw !== 'raw') || - !restOfPath.join('/').match(/\.ya?ml$/) - ) { - throw new Error('Wrong URL or Invalid file path'); - } - - // Transform to API path - const newPath = [ - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - return new URL(`${provider.apiBaseUrl}/${newPath}?ref=${ref}`); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -export function readConfig(configRoot: Config): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - // In a previous version of the configuration, we only supported github, - // and the "privateToken" key held the token to use for it. The new - // configuration method is to use the "providers" key instead. - const config = configRoot.getOptionalConfig('catalog.processors.githubApi'); - const providerConfigs = config?.getOptionalConfigArray('providers') ?? []; - const legacyToken = config?.getOptionalString('privateToken'); - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - const token = providerConfig.getOptionalString('token'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (target === 'https://github.com') { - apiBaseUrl = 'https://api.github.com'; - } else { - throw new Error( - `Provider at ${target} must configure an explicit apiBaseUrl`, - ); - } - - providers.push({ target, apiBaseUrl, token }); - } - - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.target === 'https://github.com')) { - providers.push({ - target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', - token: legacyToken, - }); - } - - return providers; -} - -/** - * A processor that adds the ability to read files from GitHub v3 APIs, such as - * the one exposed by GitHub itself. - */ -export class GithubApiReaderProcessor implements LocationProcessor { - private providers: ProviderConfig[]; - - static fromConfig(config: Config) { - return new GithubApiReaderProcessor(readConfig(config)); - } - - constructor(providers: ProviderConfig[]) { - this.providers = providers; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'github/api') { - return false; - } - - const provider = this.providers.find(p => - location.target.startsWith(`${p.target}/`), - ); - if (!provider) { - throw new Error( - `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.github.processors.githubApi.`, - ); - } - - try { - const url = getRawUrl(location.target, provider); - const options = getRequestOptions(provider); - const response = await fetch(url.toString(), options); - - if (response.ok) { - const data = await response.buffer(); - emit(result.data(location, data)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts similarity index 54% rename from plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index 48f38f0e97..e3c7611e80 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -17,18 +17,20 @@ import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { + getApiUrl, + getApiRequestOptions, getRawUrl, - getRequestOptions, - GithubApiReaderProcessor, + getRawRequestOptions, + GithubReaderProcessor, ProviderConfig, readConfig, -} from './GithubApiReaderProcessor'; +} from './GithubReaderProcessor'; -describe('GithubApiReaderProcessor', () => { - describe('getRequestOptions', () => { +describe('GithubReaderProcessor', () => { + describe('getApiRequestOptions', () => { it('sets the correct API version', () => { const config: ProviderConfig = { target: '', apiBaseUrl: '' }; - expect((getRequestOptions(config).headers as any).Accept).toEqual( + expect((getApiRequestOptions(config).headers as any).Accept).toEqual( 'application/vnd.github.v3.raw', ); }); @@ -44,24 +46,95 @@ describe('GithubApiReaderProcessor', () => { apiBaseUrl: '', }; expect( - (getRequestOptions(withToken).headers as any).Authorization, + (getApiRequestOptions(withToken).headers as any).Authorization, ).toEqual('token A'); expect( - (getRequestOptions(withoutToken).headers as any).Authorization, + (getApiRequestOptions(withoutToken).headers as any).Authorization, ).toBeUndefined(); }); }); + describe('getRawRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + target: '', + rawBaseUrl: '', + }; + expect( + (getRawRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getRawRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getApiUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { target: '', apiBaseUrl: '' }; + expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + + it('happy path for github', () => { + const config: ProviderConfig = { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getApiUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + apiBaseUrl: 'https://ghe.mycompany.net/api/v3', + }; + expect( + getApiUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ), + ); + }); + }); + describe('getRawUrl', () => { it('rejects targets that do not look like URLs', () => { const config: ProviderConfig = { target: '', apiBaseUrl: '' }; expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); - it('passes through the happy path', () => { + it('happy path for github', () => { const config: ProviderConfig = { target: 'https://github.com', - apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', }; expect( getRawUrl( @@ -70,10 +143,25 @@ describe('GithubApiReaderProcessor', () => { ), ).toEqual( new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', ), ); }); + + it('happy path for ghe', () => { + const config: ProviderConfig = { + target: 'https://ghe.mycompany.net', + rawBaseUrl: 'https://ghe.mycompany.net/raw', + }; + expect( + getRawUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'), + ); + }); }); describe('readConfig', () => { @@ -84,7 +172,7 @@ describe('GithubApiReaderProcessor', () => { { context: '', data: { - catalog: { processors: { githubApi: { providers } } }, + catalog: { processors: { github: { providers } } }, }, }, ]); @@ -93,22 +181,30 @@ describe('GithubApiReaderProcessor', () => { it('adds a default GitHub entry when missing', () => { const output = readConfig(config([])); expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, ]); }); it('injects the correct GitHub API base URL when missing', () => { const output = readConfig(config([{ target: 'https://github.com' }])); expect(output).toEqual([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, ]); }); - it('rejects custom targets with no API base URL', () => { + it('rejects custom targets with no base URLs', () => { expect(() => readConfig(config([{ target: 'https://ghe.company.com' }])), ).toThrow( - 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl', + 'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl', ); }); @@ -132,7 +228,7 @@ describe('GithubApiReaderProcessor', () => { describe('implementation', () => { it('rejects unknown types', async () => { - const processor = new GithubApiReaderProcessor([ + const processor = new GithubReaderProcessor([ { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, ]); const location: LocationSpec = { @@ -145,7 +241,7 @@ describe('GithubApiReaderProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = new GithubApiReaderProcessor([ + const processor = new GithubReaderProcessor([ { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, ]); const location: LocationSpec = { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index 9f38d782fe..f83469bd16 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,33 +15,199 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import { Config } from '@backstage/config'; +import parseGitUri from 'git-url-parse'; +import fetch, { HeadersInit, RequestInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -import { Config } from '@backstage/config'; -export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string; +/** + * The configuration parameters for a single GitHub API provider. + */ +export type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; - constructor(config?: Config) { - this.privateToken = - config?.getOptionalString('catalog.processors.github.privateToken') ?? ''; + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; } - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; + return { + headers, + }; +} - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `token ${provider.token}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://github.com/a/b/blob/branchname/path/to/c.yaml +// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); } - const requestOptions: RequestInit = { - headers, - }; + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} - return requestOptions; +// Converts for example +// from: https://github.com/a/b/blob/branchname/c.yaml +// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') || + !filepath?.match(/\.ya?ml$/) + ) { + throw new Error('Wrong URL or invalid file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + return new URL( + `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + // In a previous version of the configuration, we only supported github, + // and the "privateToken" key held the token to use for it. The new + // configuration method is to use the "providers" key instead. + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.github.providers') ?? + config.getOptionalConfigArray('catalog.processors.githubApi.providers') ?? + []; + const legacyToken = + config.getOptionalString('catalog.processors.github.privateToken') ?? + config.getOptionalString('catalog.processors.githubApi.privateToken'); + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + rawBaseUrl = 'https://raw.githubusercontent.com'; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, rawBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + token: legacyToken, + }); + } + + return providers; +} + +/** + * A processor that adds the ability to read files from GitHub v3 APIs, such as + * the one exposed by GitHub itself. + */ +export class GithubReaderProcessor implements LocationProcessor { + private providers: ProviderConfig[]; + + static fromConfig(config: Config) { + return new GithubReaderProcessor(readConfig(config)); + } + + constructor(providers: ProviderConfig[]) { + this.providers = providers; } async readLocation( @@ -49,16 +215,31 @@ export class GithubReaderProcessor implements LocationProcessor { optional: boolean, emit: LocationProcessorEmit, ): Promise { - if (location.type !== 'github') { + // The github/api type is for backward compatibility + if (location.type !== 'github' && location.type !== 'github/api') { return false; } - try { - const url = this.buildRawUrl(location.target); + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.github.providers.`, + ); + } - // TODO(freben): Should "hard" errors thrown by this line be treated as - // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + try { + const useApi = + provider.apiBaseUrl && (provider.token || !provider.rawBaseUrl); + const url = useApi + ? getApiUrl(location.target, provider) + : getRawUrl(location.target, provider); + const options = useApi + ? getApiRequestOptions(provider) + : getRawRequestOptions(provider); + console.log(url.toString()); + const response = await fetch(url.toString(), options); if (response.ok) { const data = await response.buffer(); @@ -80,41 +261,4 @@ export class GithubReaderProcessor implements LocationProcessor { return true; } - - // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } } diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index dd758eaef0..c28e0631ab 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -33,7 +33,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.1.2", + "git-url-parse": "^11.2.0", "globby": "^11.0.0", "helmet": "^4.0.0", "jsonschema": "^1.2.6", diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 47b2ebc8aa..476b321e1e 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -30,7 +30,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", - "git-url-parse": "^11.1.3", + "git-url-parse": "^11.2.0", "knex": "^0.21.1", "node-fetch": "^2.6.0", "nodegit": "^0.27.0", diff --git a/yarn.lock b/yarn.lock index e60ed587a1..9cc45f94fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10695,13 +10695,20 @@ git-up@^4.0.0: is-ssh "^1.3.0" parse-url "^5.0.0" -git-url-parse@^11.1.2, git-url-parse@^11.1.3: +git-url-parse@^11.1.2: version "11.1.3" resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.1.3.tgz#03625b6fc09905e9ad1da7bb2b84be1bf9123143" integrity sha512-GPsfwticcu52WQ+eHp0IYkAyaOASgYdtsQDIt4rUp6GbiNt1P9ddrh3O0kQB0eD4UJZszVqNT3+9Zwcg40fywA== dependencies: git-up "^4.0.0" +git-url-parse@^11.2.0: + version "11.2.0" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.2.0.tgz#2955fd51befd6d96ea1389bbe2ef57e8e6042b04" + integrity sha512-KPoHZg8v+plarZvto4ruIzzJLFQoRx+sUs5DQSr07By9IBKguVd+e6jwrFR6/TP6xrCJlNV1tPqLO1aREc7O2g== + dependencies: + git-up "^4.0.0" + gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b"