diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index b836a592ee..4dd3fd359f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { GithubReaderProcessor } from './GithubReaderProcessor'; +import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; -describe('GithubReaderProcessor', () => { +describe('GithubApiReaderProcessor', () => { it('should build raw api', () => { - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); const tests = [ { @@ -82,7 +82,7 @@ describe('GithubReaderProcessor', () => { for (const test of tests) { process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts new file mode 100644 index 0000000000..ff61d004ca --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class GithubApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index a0e089aaae..b83c9a16f3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,29 +15,11 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; - - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - async readLocation( location: LocationSpec, optional: boolean, @@ -52,7 +34,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + const response = await fetch(url.toString()); if (response.ok) { const data = await response.buffer(); @@ -76,9 +58,9 @@ export class GithubReaderProcessor implements LocationProcessor { } // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { + // from: https://github.com/a/b/blob/master/c.yaml + // to: https://raw.githubusercontent.com/a/b/master/c.yaml + private buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -87,7 +69,6 @@ export class GithubReaderProcessor implements LocationProcessor { userOrOrg, repoName, blobKeyword, - ref, ...restOfPath ] = url.pathname.split('/'); @@ -99,21 +80,13 @@ export class GithubReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL or Invalid file path'); + throw new Error('Wrong GitHub URL'); } - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; url.protocol = 'https'; - url.search = `ref=${ref}`; return url; } catch (e) {