diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index b84267c854..7b62e5579e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -46,6 +46,7 @@ "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", + "msw": "^0.19.5", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 545465cde0..e5ef410456 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -29,6 +29,7 @@ import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; +import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; import { @@ -62,6 +63,7 @@ export class LocationReaders implements LocationReader { new GithubApiReaderProcessor(), new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), + new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts new file mode 100644 index 0000000000..e30b320bf4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReaderProcessor } from './UrlReaderProcessor'; +import { + LocationProcessorDataResult, + LocationProcessorResult, + LocationProcessorErrorResult, +} from './types'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; + +describe('UrlReaderProcessor', () => { + const mockApiOrigin = 'http://localhost:23000'; + const server = setupServer(); + + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + it('should load from url', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) => + res(ctx.body('Hello')), + ), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorDataResult; + + expect(generated.type).toBe('data'); + expect(generated.location).toBe(spec); + expect(generated.data.toString('utf8')).toBe('Hello'); + }); + + it('should fail load from url with error', async () => { + const processor = new UrlReaderProcessor(); + const spec = { + type: 'url', + target: `${mockApiOrigin}/component-notfound.yaml`, + }; + + server.use( + rest.get(`${mockApiOrigin}/component-notfound.yaml`, (_, res, ctx) => { + return res(ctx.status(404)); + }), + ); + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as LocationProcessorErrorResult; + + expect(generated.type).toBe('error'); + expect(generated.location).toBe(spec); + expect(generated.error.name).toBe('NotFoundError'); + expect(generated.error.message).toBe( + `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`, + ); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts new file mode 100644 index 0000000000..0c879ea70c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class UrlReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'url') { + return false; + } + + try { + const response = await fetch(location.target); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/setupTests.ts b/plugins/catalog-backend/src/setupTests.ts index f7b6ca962d..ba33cf996b 100644 --- a/plugins/catalog-backend/src/setupTests.ts +++ b/plugins/catalog-backend/src/setupTests.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -require('jest-fetch-mock').enableMocks(); - export {};