feat: add UrlReaderProcessor to consume entities from a simple URL

plugins/catalog-backend/package.json

@@ -46,6 +46,7 @@
     "@types/uuid": "^8.0.0",
     "@types/yup": "^0.28.2",
     "jest-fetch-mock": "^3.0.3",
+    "msw": "^0.19.5",
     "supertest": "^4.0.2"
   },
   "files": [

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -28,6 +28,7 @@ import { FileReaderProcessor } from './processors/FileReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
 import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
+import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
 import * as result from './processors/results';
 import {
@@ -60,6 +61,7 @@ export class LocationReaders implements LocationReader {
       new GithubReaderProcessor(),
       new GithubApiReaderProcessor(),
       new GitlabReaderProcessor(),
+      new UrlReaderProcessor(),
       new YamlProcessor(),
       new EntityPolicyProcessor(entityPolicy),
       new LocationRefProcessor(),

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReaderProcessor } from './UrlReaderProcessor';
+import {
+  LocationProcessorDataResult,
+  LocationProcessorResult,
+  LocationProcessorErrorResult,
+} from './types';
+import { setupServer } from 'msw/node';
+import { rest } from 'msw';
+
+describe('UrlReaderProcessor', () => {
+  const mockApiOrigin = 'http://localhost:23000';
+  const server = setupServer();
+
+  beforeAll(() => server.listen());
+  afterEach(() => server.resetHandlers());
+  afterAll(() => server.close());
+
+  it('should load from url', async () => {
+    const processor = new UrlReaderProcessor();
+    const spec = {
+      type: 'url',
+      target: `${mockApiOrigin}/component.yaml`,
+    };
+
+    server.use(
+      rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) =>
+        res(ctx.body('Hello')),
+      ),
+    );
+
+    const generated = (await new Promise<LocationProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit),
+    )) as LocationProcessorDataResult;
+
+    expect(generated.type).toBe('data');
+    expect(generated.location).toBe(spec);
+    expect(generated.data.toString('utf8')).toBe('Hello');
+  });
+
+  it('should fail load from url with error', async () => {
+    const processor = new UrlReaderProcessor();
+    const spec = {
+      type: 'url',
+      target: `${mockApiOrigin}/component-notfound.yaml`,
+    };
+
+    server.use(
+      rest.get(`${mockApiOrigin}/component-notfound.yaml`, (_, res, ctx) => {
+        return res(ctx.status(404));
+      }),
+    );
+
+    const generated = (await new Promise<LocationProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit),
+    )) as LocationProcessorErrorResult;
+
+    expect(generated.type).toBe('error');
+    expect(generated.location).toBe(spec);
+    expect(generated.error.name).toBe('NotFoundError');
+    expect(generated.error.message).toBe(
+      `${mockApiOrigin}/component-notfound.yaml could not be read, 404 Not Found`,
+    );
+  });
+});

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import fetch from 'node-fetch';
+import * as result from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+
+export class UrlReaderProcessor implements LocationProcessor {
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'url') {
+      return false;
+    }
+
+    try {
+      const response = await fetch(location.target);
+
+      if (response.ok) {
+        const data = await response.buffer();
+        emit(result.data(location, data));
+      } else {
+        const message = `${location.target} could not be read, ${response.status} ${response.statusText}`;
+        if (response.status === 404) {
+          if (!optional) {
+            emit(result.notFoundError(location, message));
+          }
+        } else {
+          emit(result.generalError(location, message));
+        }
+      }
+    } catch (e) {
+      const message = `Unable to read ${location.type} ${location.target}, ${e}`;
+      emit(result.generalError(location, message));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend/src/setupTests.ts

@@ -14,6 +14,4 @@
  * limitations under the License.
  */
 
-require('jest-fetch-mock').enableMocks();
-
 export {};

yarn.lock

@@ -13127,7 +13127,7 @@ ms@^2.0.0, ms@^2.1.1:
   resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
-msw@^0.19.0:
+msw@^0.19.0, msw@^0.19.5:
   version "0.19.5"
   resolved "https://registry.npmjs.org/msw/-/msw-0.19.5.tgz#7d2a1a852ccf1644d3db6735d69fff6777aac33f"
   integrity sha512-J5eQ++gDVZoHPC8gVXtWcakLjgmPipvFj/sEnlRV/WViXuiq2CamSqO3Wbh6H8bAmj+k2vUWCfcVT1HjMdKB2Q==