From 2e605f91729b3260bd4f59eda7a3fcb5640f2e68 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 23 Jul 2020 10:49:40 +0200 Subject: [PATCH] feat: add UrlReaderProcessor to consume entities from a simple URL --- plugins/catalog-backend/package.json | 1 + .../src/ingestion/LocationReaders.ts | 2 + .../processors/UrlReaderProcessor.test.ts | 80 +++++++++++++++++++ .../processors/UrlReaderProcessor.ts | 55 +++++++++++++ plugins/catalog-backend/src/setupTests.ts | 2 - yarn.lock | 2 +- 6 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9c7a18087d..b342cae3db 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -46,6 +46,7 @@ "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", + "msw": "^0.19.5", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 8ec575d6ec..2aa41f6c50 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -28,6 +28,7 @@ import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; +import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; import { @@ -60,6 +61,7 @@ export class LocationReaders implements LocationReader { new GithubReaderProcessor(), new GithubApiReaderProcessor(), new GitlabReaderProcessor(), + new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts new file mode 100644 index 0000000000..e30b320bf4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReaderProcessor } from './UrlReaderProcessor'; +import { + LocationProcessorDataResult, + LocationProcessorResult, + LocationProcessorErrorResult, +} from './types'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; + +describe('UrlReaderProcessor', () => { + const mockApiOrigin = 'http://localhost:23000'; + const server = setupServer(); + + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + it('should load from url', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => + res(ctx.body('Hello')), + ), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorDataResult; + + expect(generated.type).toBe('data'); + expect(generated.location).toBe(spec); + expect(generated.data.toString('utf8')).toBe('Hello'); + }); + + it('should fail load from url with error', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component-notfound.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component-notfound.yaml`, (_, res, ctx) => { + return res(ctx.status(404)); + }), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorErrorResult; + + expect(generated.type).toBe('error'); + expect(generated.location).toBe(spec); + expect(generated.error.name).toBe('NotFoundError'); + expect(generated.error.message).toBe( + `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`, + ); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts new file mode 100644 index 0000000000..0c879ea70c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class UrlReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'url') { + return false; + } + + try { + const response = await fetch(location.target); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/setupTests.ts b/plugins/catalog-backend/src/setupTests.ts index f7b6ca962d..ba33cf996b 100644 --- a/plugins/catalog-backend/src/setupTests.ts +++ b/plugins/catalog-backend/src/setupTests.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -require('jest-fetch-mock').enableMocks(); - export {}; diff --git a/yarn.lock b/yarn.lock index 6c0ff99388..ae7f5feabf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13127,7 +13127,7 @@ ms@^2.0.0, ms@^2.1.1: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -msw@^0.19.0: +msw@^0.19.0, msw@^0.19.5: version "0.19.5" resolved "https://registry.npmjs.org/msw/-/msw-0.19.5.tgz#7d2a1a852ccf1644d3db6735d69fff6777aac33f" integrity sha512-J5eQ++gDVZoHPC8gVXtWcakLjgmPipvFj/sEnlRV/WViXuiq2CamSqO3Wbh6H8bAmj+k2vUWCfcVT1HjMdKB2Q==