diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 50915bca51..9c0b066f4f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@types/node-fetch": "^2.5.7", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -25,6 +26,7 @@ "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", + "node-fetch": "^2.6.0", "sqlite3": "^4.2.0", "uuid": "^8.0.0", "winston": "^3.2.1", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 676c318086..80f58aed6a 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -15,6 +15,7 @@ */ import { FileLocationSource } from './sources/FileLocationSource'; +import { GitHubLocationSource } from './sources/GitHubLocationSource'; import { LocationSource } from './sources/types'; import { LocationReader, ReaderOutput } from './types'; @@ -22,6 +23,7 @@ export class LocationReaders implements LocationReader { static create(): LocationReader { return new LocationReaders({ file: new FileLocationSource(), + github: new GitHubLocationSource(), }); } diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts new file mode 100644 index 0000000000..ac0a2faa23 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { ReaderOutput } from '../types'; +import { LocationSource } from './types'; +import { readDescriptorYaml } from './util'; +import { URL } from 'url'; + +// Pointing to raw.githubusercontent.com for now +// to be changed in the future, after auth and tokens are done +export class GitHubLocationSource implements LocationSource { + async read(target: string): Promise { + let url: URL; + + try { + url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; + url.protocol = 'https'; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + + let rawYaml; + try { + rawYaml = await fetch(url.toString()).then((x) => { + return x.text(); + }); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + + try { + return readDescriptorYaml(rawYaml); + } catch (e) { + throw new Error(`Malformed descriptor at "${target}", ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts new file mode 100644 index 0000000000..083c2f7e86 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('node-fetch'); + +import fs from 'fs-extra'; +import fetch from 'node-fetch'; +import path from 'path'; +import { GitHubLocationSource } from '../GitHubLocationSource'; + +const { Response } = jest.requireActual('node-fetch'); + +const FIXTURES_DIR = path.resolve( + __dirname, + '..', + '..', + '..', + '..', + 'fixtures', +); +const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => { + acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8'); + return acc; +}, {} as Record); + +describe('Unit: GitHubLocationSource', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('fetches the file and parses it correctly', async () => { + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures['one_component.yaml'])), + ); + const reader = new GitHubLocationSource(); + + const result = await reader.read( + 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', + ); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); + + it('changes the url to point to https://raw.githubusercontent.com', async () => { + const gitHubUrl = `https://github.com`; + const project = `spotify/backstage`; + const folderPath = `master/plugins/catalog-backend/fixtures`; + const componentFilename = `one_component.yaml`; + const rawGitHubUrl = `https://raw.githubusercontent.com`; + const reader = new GitHubLocationSource(); + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures[componentFilename])), + ); + + await reader.read( + `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, + ); + + expect(fetch).toHaveBeenCalledWith( + `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`, + ); + }); + + describe('rejects wrong urls', () => { + const reader = new GitHubLocationSource(); + + it.each([ + ['http://example.com/one_component.yaml'], + ['http://github.com/one_component.yaml'], + ['http://github.com/PROJECT/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.json'], + ])( + '%p', + async (url: string) => + await expect(reader.read(url)).rejects.toThrow(/url/), + ); + }); +}); + +describe('Integration: GitHubLocationSource', () => { + beforeAll(() => { + (fetch as any).mockImplementation(jest.requireActual('node-fetch')); + }); + + it('fetches the fixture from backstage repo', async () => { + const PERMANENT_LINK = + 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; + const reader = new GitHubLocationSource(); + + const result = await reader.read(PERMANENT_LINK); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 08773708ab..c4f92768a6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4129,6 +4129,14 @@ dependencies: "@types/express" "*" +"@types/node-fetch@^2.5.7": + version "2.5.7" + resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" + integrity sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2": version "13.9.2" resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"