From e9773a0bdb801dd58663448688e2a875c8b4aae0 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 01:11:20 +0700 Subject: [PATCH 1/6] catalog(github): add github v3 reader processor --- .../src/ingestion/LocationReaders.ts | 2 + .../GithubV3ReaderProcessor.test.ts | 21 ++++ .../processors/GithubV3ReaderProcessor.ts | 119 ++++++++++++++++++ yarn.lock | 7 -- 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0fe87440fb..7d11659bf4 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GithubV3ReaderProcessor } from './processors/GithubV3ReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -57,6 +58,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GithubV3ReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts new file mode 100644 index 0000000000..1b39da2753 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor'; + +describe('GithubV3ReaderProcessor', () => { + // it('should build raw url') +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts new file mode 100644 index 0000000000..2316c36629 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +const privateToken = process.env.GITHUB_PRIVATE_TOKEN; + +export class GithubV3ReaderProcessor implements LocationProcessor { + getRequestOptions(): RequestInit { + const requestOptions: RequestInit = { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }; + + if (privateToken) { + requestOptions.headers.Authorization = `token ${privateToken}`; + } + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/v3') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const buffer = await response.buffer(); + + emit(result.data(location, buffer)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/yarn.lock b/yarn.lock index 8f5d9e8cec..0641f5ed77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3960,13 +3960,6 @@ "@types/node" "*" rollup "^0.63.4" -"@types/sanitize-html@^1.23.3": - version "1.23.3" - resolved "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.23.3.tgz#26527783aba3bf195ad8a3c3e51bd3713526fc0d" - integrity sha512-Isg8N0ifKdDq6/kaNlIcWfapDXxxquMSk2XC5THsOICRyOIhQGds95XH75/PL/g9mExi4bL8otIqJM/Wo96WxA== - dependencies: - htmlparser2 "^4.1.0" - "@types/serve-static@*": version "1.13.3" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1" From c295d8ce0687ba4dbac0853002dc7de2cb787685 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 12:14:28 +0700 Subject: [PATCH 2/6] catalog(github): merge github v3 to github --- .../src/ingestion/LocationReaders.ts | 1 - .../processors/GithubReaderProcessor.test.ts | 65 ++++++++++ .../processors/GithubReaderProcessor.ts | 44 +++++-- .../GithubV3ReaderProcessor.test.ts | 21 ---- .../processors/GithubV3ReaderProcessor.ts | 119 ------------------ 5 files changed, 100 insertions(+), 150 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 7d11659bf4..bbdc7d8933 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -58,7 +58,6 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), - new GithubV3ReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts new file mode 100644 index 0000000000..5d64caf5b9 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubReaderProcessor } from './GithubReaderProcessor'; + +describe('GithubReaderProcessor', () => { + it('should build raw api', () => { + const processor = new GithubReaderProcessor(); + + const tests = [ + { + target: 'https://github.com/a/b/blob/master/path/to/c.yaml', + url: new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + target: + 'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml', + url: new URL( + 'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master', + ), + err: undefined, + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else { + expect(processor.buildRawUrl(test.target)).toEqual(test.url); + } + } + }); + + it('should return request options', () => { + // todo + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index b83c9a16f3..11610234bd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,11 +15,28 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +const privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + export class GithubReaderProcessor implements LocationProcessor { + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + if (privateToken) { + headers.Authorization = `token ${privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + async readLocation( location: LocationSpec, optional: boolean, @@ -34,7 +51,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString()); + const response = await fetch(url.toString(), this.getRequestOptions()); if (response.ok) { const data = await response.buffer(); @@ -58,9 +75,9 @@ export class GithubReaderProcessor implements LocationProcessor { } // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -69,6 +86,7 @@ export class GithubReaderProcessor implements LocationProcessor { userOrOrg, repoName, blobKeyword, + ref, ...restOfPath ] = url.pathname.split('/'); @@ -80,13 +98,21 @@ export class GithubReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL'); + throw new Error('Wrong GitHub URL or Invalid file path'); } - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; url.protocol = 'https'; + url.search = `ref=${ref}`; return url; } catch (e) { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts deleted file mode 100644 index 1b39da2753..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor'; - -describe('GithubV3ReaderProcessor', () => { - // it('should build raw url') -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts deleted file mode 100644 index 2316c36629..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -const privateToken = process.env.GITHUB_PRIVATE_TOKEN; - -export class GithubV3ReaderProcessor implements LocationProcessor { - getRequestOptions(): RequestInit { - const requestOptions: RequestInit = { - headers: { - Accept: 'application/vnd.github.v3.raw', - }, - }; - - if (privateToken) { - requestOptions.headers.Authorization = `token ${privateToken}`; - } - return requestOptions; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'github/v3') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - if (response.ok) { - const buffer = await response.buffer(); - - emit(result.data(location, buffer)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } - - // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ref, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; - url.protocol = 'https'; - url.search = `ref=${ref}`; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} From af33400021dae4d0f699a13f3b436bbd7f2fee05 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 12:22:46 +0700 Subject: [PATCH 3/6] catalog(github): fix test --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 1 - .../src/ingestion/processors/GithubReaderProcessor.test.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index bbdc7d8933..0fe87440fb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,7 +26,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubV3ReaderProcessor } from './processors/GithubV3ReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index 5d64caf5b9..0b3057f97e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -32,7 +32,7 @@ describe('GithubReaderProcessor', () => { target: 'https://api.com/a/b/blob/master/path/to/c.yaml', url: null, err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL', + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL or Invalid file path', }, { target: 'com/a/b/blob/master/path/to/c.yaml', From 48addc9259367cc5de2700bba57e17fe8bc7b322 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 14:57:03 +0700 Subject: [PATCH 4/6] catalog(github): add test get request options --- .../processors/GithubReaderProcessor.test.ts | 26 ++++++++++++++++++- .../processors/GithubReaderProcessor.ts | 9 ++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index 0b3057f97e..b836a592ee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -60,6 +60,30 @@ describe('GithubReaderProcessor', () => { }); it('should return request options', () => { - // todo + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Accept: 'application/vnd.github.v3.raw', + Authorization: 'token 0123456789', + }, + }, + }, + { + token: '', + expect: { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }, + }, + ]; + + for (const test of tests) { + process.env.GITHUB_PRIVATE_TOKEN = test.token; + const processor = new GithubReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index 11610234bd..a0e089aaae 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -19,15 +19,16 @@ import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -const privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; - export class GithubReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + getRequestOptions(): RequestInit { const headers: HeadersInit = { Accept: 'application/vnd.github.v3.raw', }; - if (privateToken) { - headers.Authorization = `token ${privateToken}`; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; } const requestOptions: RequestInit = { From d4d9c33ca32d9e0f60664c8e2d43fa1958e5dcf4 Mon Sep 17 00:00:00 2001 From: danztran Date: Thu, 9 Jul 2020 23:59:56 +0700 Subject: [PATCH 5/6] catalog(github): split to github api reader processor --- ...st.ts => GithubApiReaderProcessor.test.ts} | 8 +- .../processors/GithubApiReaderProcessor.ts | 121 ++++++++++++++++++ .../processors/GithubReaderProcessor.ts | 45 ++----- 3 files changed, 134 insertions(+), 40 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/{GithubReaderProcessor.test.ts => GithubApiReaderProcessor.test.ts} (91%) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index b836a592ee..4dd3fd359f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { GithubReaderProcessor } from './GithubReaderProcessor'; +import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; -describe('GithubReaderProcessor', () => { +describe('GithubApiReaderProcessor', () => { it('should build raw api', () => { - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); const tests = [ { @@ -82,7 +82,7 @@ describe('GithubReaderProcessor', () => { for (const test of tests) { process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts new file mode 100644 index 0000000000..ff61d004ca --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class GithubApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index a0e089aaae..b83c9a16f3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,29 +15,11 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; - - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - async readLocation( location: LocationSpec, optional: boolean, @@ -52,7 +34,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + const response = await fetch(url.toString()); if (response.ok) { const data = await response.buffer(); @@ -76,9 +58,9 @@ export class GithubReaderProcessor implements LocationProcessor { } // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { + // from: https://github.com/a/b/blob/master/c.yaml + // to: https://raw.githubusercontent.com/a/b/master/c.yaml + private buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -87,7 +69,6 @@ export class GithubReaderProcessor implements LocationProcessor { userOrOrg, repoName, blobKeyword, - ref, ...restOfPath ] = url.pathname.split('/'); @@ -99,21 +80,13 @@ export class GithubReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL or Invalid file path'); + throw new Error('Wrong GitHub URL'); } - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; url.protocol = 'https'; - url.search = `ref=${ref}`; return url; } catch (e) { From 77b5160254397d379b649c97f2f6c68626e9f083 Mon Sep 17 00:00:00 2001 From: danztran Date: Fri, 10 Jul 2020 00:05:44 +0700 Subject: [PATCH 6/6] catalog(github): add github api to default processors --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0fe87440fb..8ec575d6ec 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -57,6 +58,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GithubApiReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy),