diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0fe87440fb..7d11659bf4 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GithubV3ReaderProcessor } from './processors/GithubV3ReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -57,6 +58,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GithubV3ReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts new file mode 100644 index 0000000000..1b39da2753 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor'; + +describe('GithubV3ReaderProcessor', () => { + // it('should build raw url') +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts new file mode 100644 index 0000000000..2316c36629 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +const privateToken = process.env.GITHUB_PRIVATE_TOKEN; + +export class GithubV3ReaderProcessor implements LocationProcessor { + getRequestOptions(): RequestInit { + const requestOptions: RequestInit = { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }; + + if (privateToken) { + requestOptions.headers.Authorization = `token ${privateToken}`; + } + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/v3') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const buffer = await response.buffer(); + + emit(result.data(location, buffer)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/yarn.lock b/yarn.lock index 8f5d9e8cec..0641f5ed77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3960,13 +3960,6 @@ "@types/node" "*" rollup "^0.63.4" -"@types/sanitize-html@^1.23.3": - version "1.23.3" - resolved "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.23.3.tgz#26527783aba3bf195ad8a3c3e51bd3713526fc0d" - integrity sha512-Isg8N0ifKdDq6/kaNlIcWfapDXxxquMSk2XC5THsOICRyOIhQGds95XH75/PL/g9mExi4bL8otIqJM/Wo96WxA== - dependencies: - htmlparser2 "^4.1.0" - "@types/serve-static@*": version "1.13.3" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1"