diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 50915bca51..9c0b066f4f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@types/node-fetch": "^2.5.7", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -25,6 +26,7 @@ "knex": "^0.21.1", "lodash": "^4.17.15", "morgan": "^1.10.0", + "node-fetch": "^2.6.0", "sqlite3": "^4.2.0", "uuid": "^8.0.0", "winston": "^3.2.1", diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts new file mode 100644 index 0000000000..9538698544 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { ReaderOutput } from '../types'; +import { LocationSource } from './types'; +import { readDescriptorYaml } from './util'; +import { URL } from 'url'; + +// Pointing to raw.githubusercontent.com for now +// to be changed in the future, after auth and tokens are done +export class GitHubLocationSource implements LocationSource { + async read(target: string): Promise { + let url: URL; + + try { + url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error(); + } + + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; + url.protocol = 'https'; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + + let rawYaml; + try { + rawYaml = await fetch(url.toString()).then(x => { + return x.text(); + }); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + + try { + return readDescriptorYaml(rawYaml); + } catch (e) { + throw new Error(`Malformed descriptor at "${target}", ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts new file mode 100644 index 0000000000..6a22570309 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts @@ -0,0 +1,94 @@ +jest.mock('node-fetch'); + +import fs from 'fs-extra'; +import fetch from 'node-fetch'; +import path from 'path'; +import { GitHubLocationSource } from '../GitHubLocationSource'; + +const { Response } = jest.requireActual('node-fetch'); + +const fixtures_dir = path.resolve( + __dirname, + '..', + '..', + '..', + '..', + 'fixtures', +); +const fixtures = fs.readdirSync(fixtures_dir).reduce((acc, filename) => { + acc[filename] = fs.readFileSync(path.resolve(fixtures_dir, filename), 'utf8'); + return acc; +}, {} as Record); + +describe('Unit: GitHubLocationSource', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('fetches the file and parses it correctly', async () => { + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures['one_component.yaml'])), + ); + const reader = new GitHubLocationSource(); + + const result = await reader.read( + 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', + ); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); + + it('changes the url to point to https://raw.githubusercontent.com', async () => { + const gitHubUrl = `https://github.com`; + const project = `spotify/backstage`; + const path = `master/plugins/catalog-backend/fixtures`; + const componentFilename = `one_component.yaml`; + const rawGitHubUrl = `https://raw.githubusercontent.com`; + const reader = new GitHubLocationSource(); + (fetch as any).mockReturnValueOnce( + Promise.resolve(new Response(fixtures[componentFilename])), + ); + + await reader.read( + `${gitHubUrl}/${project}/blob/${path}/${componentFilename}`, + ); + + expect(fetch).toHaveBeenCalledWith( + `${rawGitHubUrl}/${project}/${path}/${componentFilename}`, + ); + }); + + describe('rejects wrong urls', () => { + const reader = new GitHubLocationSource(); + + it.each([ + ['http://example.com/one_component.yaml'], + ['http://github.com/one_component.yaml'], + ['http://github.com/PROJECT/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.json'], + ])( + '%p', + async (url: string) => + await expect(reader.read(url)).rejects.toThrow(/url/), + ); + }); +}); + +describe('Integration: GitHubLocationSource', () => { + beforeAll(() => { + (fetch as any).mockImplementation(jest.requireActual('node-fetch')); + }); + + it('fetches the fixture from backstage repo', async () => { + const PERMANENT_LINK = + 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; + const reader = new GitHubLocationSource(); + + const result = await reader.read(PERMANENT_LINK); + + expect(result[0].type).toBe('data'); + expect((result[0] as any).data.metadata.name).toBe('component3'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 0d2cda2b05..f5cb15f5b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4093,6 +4093,14 @@ dependencies: "@types/express" "*" +"@types/node-fetch@^2.5.7": + version "2.5.7" + resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" + integrity sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2": version "13.9.2" resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" @@ -6823,7 +6831,7 @@ columnify@^1.5.4, columnify@~1.5.4: strip-ansi "^3.0.0" wcwidth "^1.0.0" -combined-stream@^1.0.6, combined-stream@~1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -9717,6 +9725,15 @@ form-data@^2.3.1: combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" + integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"