From ca83d975919dbef49a8043050e68be9c2321d89a Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 20 May 2020 15:21:24 +0200 Subject: [PATCH 1/4] feat: github location reader --- plugins/catalog-backend/package.json | 2 + .../ingestion/sources/GitHubLocationSource.ts | 74 +++++++++++++++ .../__tests__/GitHubLocationSource.test.ts | 94 +++++++++++++++++++ yarn.lock | 19 +++- 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts create mode 100644 plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 50915bca51..9c0b066f4f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@types/node-fetch": "^2.5.7", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -25,6 +26,7 @@ "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", + "node-fetch": "^2.6.0", "sqlite3": "^4.2.0", "uuid": "^8.0.0", "winston": "^3.2.1", diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts new file mode 100644 index 0000000000..9538698544 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { ReaderOutput } from '../types'; +import { LocationSource } from './types'; +import { readDescriptorYaml } from './util'; +import { URL } from 'url'; + +// Pointing to raw.githubusercontent.com for now +// to be changed in the future, after auth and tokens are done +export class GitHubLocationSource implements LocationSource { + async read(target: string): Promise { + let url: URL; + + try { + url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error(); + } + + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; + url.protocol = 'https'; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + + let rawYaml; + try { + rawYaml = await fetch(url.toString()).then(x => { + return x.text(); + }); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + + try { + return readDescriptorYaml(rawYaml); + } catch (e) { + throw new Error(`Malformed descriptor at "${target}", ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts new file mode 100644 index 0000000000..6a22570309 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts @@ -0,0 +1,94 @@ +jest.mock('node-fetch'); + +import fs from 'fs-extra'; +import fetch from 'node-fetch'; +import path from 'path'; +import { GitHubLocationSource } from '../GitHubLocationSource'; + +const { Response } = jest.requireActual('node-fetch'); + +const fixtures_dir = path.resolve( + __dirname, + '..', + '..', + '..', + '..', + 'fixtures', +); +const fixtures = fs.readdirSync(fixtures_dir).reduce((acc, filename) => { + acc[filename] = fs.readFileSync(path.resolve(fixtures_dir, filename), 'utf8'); + return acc; +}, {} as Record); + +describe('Unit: GitHubLocationSource', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('fetches the file and parses it correctly', async () => { + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures['one_component.yaml'])), + ); + const reader = new GitHubLocationSource(); + + const result = await reader.read( + 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', + ); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); + + it('changes the url to point to https://raw.githubusercontent.com', async () => { + const gitHubUrl = `https://github.com`; + const project = `spotify/backstage`; + const path = `master/plugins/catalog-backend/fixtures`; + const componentFilename = `one_component.yaml`; + const rawGitHubUrl = `https://raw.githubusercontent.com`; + const reader = new GitHubLocationSource(); + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures[componentFilename])), + ); + + await reader.read( + `${gitHubUrl}/${project}/blob/${path}/${componentFilename}`, + ); + + expect(fetch).toHaveBeenCalledWith( + `${rawGitHubUrl}/${project}/${path}/${componentFilename}`, + ); + }); + + describe('rejects wrong urls', () => { + const reader = new GitHubLocationSource(); + + it.each([ + ['http://example.com/one_component.yaml'], + ['http://github.com/one_component.yaml'], + ['http://github.com/PROJECT/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.json'], + ])( + '%p', + async (url: string) => + await expect(reader.read(url)).rejects.toThrow(/url/), + ); + }); +}); + +describe('Integration: GitHubLocationSource', () => { + beforeAll(() => { + (fetch as any).mockImplementation(jest.requireActual('node-fetch')); + }); + + it('fetches the fixture from backstage repo', async () => { + const PERMANENT_LINK = + 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; + const reader = new GitHubLocationSource(); + + const result = await reader.read(PERMANENT_LINK); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 0d2cda2b05..f5cb15f5b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4093,6 +4093,14 @@ dependencies: "@types/express" "*" +"@types/node-fetch@^2.5.7": + version "2.5.7" + resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" + integrity sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2": version "13.9.2" resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" @@ -6823,7 +6831,7 @@ columnify@^1.5.4, columnify@~1.5.4: strip-ansi "^3.0.0" wcwidth "^1.0.0" -combined-stream@^1.0.6, combined-stream@~1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -9717,6 +9725,15 @@ form-data@^2.3.1: combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" + integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" From 8f0e5c3f123018ae008eb3dcfe927e94fa5259f7 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 20 May 2020 15:25:13 +0200 Subject: [PATCH 2/4] feat: add github reader type --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 2 ++ .../src/ingestion/sources/GitHubLocationSource.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 676c318086..80f58aed6a 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -15,6 +15,7 @@ */ import { FileLocationSource } from './sources/FileLocationSource'; +import { GitHubLocationSource } from './sources/GitHubLocationSource'; import { LocationSource } from './sources/types'; import { LocationReader, ReaderOutput } from './types'; @@ -22,6 +23,7 @@ export class LocationReaders implements LocationReader { static create(): LocationReader { return new LocationReaders({ file: new FileLocationSource(), + github: new GitHubLocationSource(), }); } diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts index 9538698544..a16cc9ae56 100644 --- a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts @@ -58,7 +58,7 @@ export class GitHubLocationSource implements LocationSource { let rawYaml; try { - rawYaml = await fetch(url.toString()).then(x => { + rawYaml = await fetch(url.toString()).then((x) => { return x.text(); }); } catch (e) { From 4fcfc477a48a2e69569abbc556fcef55cdc2c496 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 20 May 2020 15:37:53 +0200 Subject: [PATCH 3/4] fix: lint stuff --- .../__tests__/GitHubLocationSource.test.ts | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts index 6a22570309..083c2f7e86 100644 --- a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts +++ b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts @@ -1,3 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + jest.mock('node-fetch'); import fs from 'fs-extra'; @@ -7,7 +23,7 @@ import { GitHubLocationSource } from '../GitHubLocationSource'; const { Response } = jest.requireActual('node-fetch'); -const fixtures_dir = path.resolve( +const FIXTURES_DIR = path.resolve( __dirname, '..', '..', @@ -15,8 +31,8 @@ const fixtures_dir = path.resolve( '..', 'fixtures', ); -const fixtures = fs.readdirSync(fixtures_dir).reduce((acc, filename) => { - acc[filename] = fs.readFileSync(path.resolve(fixtures_dir, filename), 'utf8'); +const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => { + acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8'); return acc; }, {} as Record); @@ -42,7 +58,7 @@ describe('Unit: GitHubLocationSource', () => { it('changes the url to point to https://raw.githubusercontent.com', async () => { const gitHubUrl = `https://github.com`; const project = `spotify/backstage`; - const path = `master/plugins/catalog-backend/fixtures`; + const folderPath = `master/plugins/catalog-backend/fixtures`; const componentFilename = `one_component.yaml`; const rawGitHubUrl = `https://raw.githubusercontent.com`; const reader = new GitHubLocationSource(); @@ -51,11 +67,11 @@ describe('Unit: GitHubLocationSource', () => { ); await reader.read( - `${gitHubUrl}/${project}/blob/${path}/${componentFilename}`, + `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, ); expect(fetch).toHaveBeenCalledWith( - `${rawGitHubUrl}/${project}/${path}/${componentFilename}`, + `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`, ); }); From 2f01a5be2663206da94d797d5a342f785908a09e Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 25 May 2020 10:21:59 +0200 Subject: [PATCH 4/4] fix: error message --- .../src/ingestion/sources/GitHubLocationSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts index a16cc9ae56..ac0a2faa23 100644 --- a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts @@ -45,7 +45,7 @@ export class GitHubLocationSource implements LocationSource { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error(); + throw new Error('Wrong GitHub URL'); } // Removing the "blob" part