Merge pull request #869 from spotify/freben/refresh-split

Separate out and generalise concerns of reading and parsing

packages/backend/src/plugins/catalog.ts

@@ -15,12 +15,28 @@
  */
 
 import {
-  DatabaseCatalog,
+  DatabaseItemsCatalog,
+  DatabaseLocationsCatalog,
+  DatabaseManager,
   createRouter,
+  runPeriodically,
+  LocationReaders,
+  DescriptorParsers,
 } from '@backstage/plugin-catalog-backend';
 import { PluginEnvironment } from '../types';
 
-export default async function({ logger, database }: PluginEnvironment) {
-  const catalog = await DatabaseCatalog.create(database, logger);
-  return await createRouter({ catalog, logger });
+export default async function ({ logger, database }: PluginEnvironment) {
+  const reader = LocationReaders.create();
+  const parser = DescriptorParsers.create();
+
+  const db = await DatabaseManager.createDatabase(database);
+  runPeriodically(
+    () => DatabaseManager.refreshLocations(db, reader, parser, logger),
+    10000,
+  );
+
+  const itemsCatalog = new DatabaseItemsCatalog(db);
+  const locationsCatalog = new DatabaseLocationsCatalog(db);
+
+  return await createRouter({ itemsCatalog, locationsCatalog, logger });
 }

plugins/catalog-backend/fixtures/one_component.yaml

@@ -1,6 +1,6 @@
 apiVersion: catalog.backstage.io/v1
 kind: Component
 metadata:
-  name: component1
+  name: component3
 spec:
   type: service

plugins/catalog-backend/fixtures/two_components.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: catalog.backstage.io/v1
+kind: Component
+metadata:
+  name: component1
+spec:
+  type: service
+---
+apiVersion: catalog.backstage.io/v1
+kind: Component
+metadata:
+  name: component2
+spec:
+  type: service

plugins/catalog-backend/src/catalog/CatalogLogic.test.ts

@@ -1,75 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { PassThrough } from 'stream';
-import winston from 'winston';
-import { CatalogLogic } from './CatalogLogic';
-import { Catalog } from './types';
-
-describe('CatalogLogic', () => {
-  const logger = winston.createLogger({
-    transports: [new winston.transports.Stream({ stream: new PassThrough() })],
-  });
-
-  describe('refreshLocations', () => {
-    it('works with no locations added', async () => {
-      const catalog = ({
-        addOrUpdateComponent: jest.fn(),
-        locations: jest.fn().mockResolvedValue([]),
-      } as unknown) as Catalog;
-      const locationReader = jest.fn();
-
-      await expect(
-        CatalogLogic.refreshLocations(catalog, locationReader, logger),
-      ).resolves.toBeUndefined();
-      expect(locationReader).not.toHaveBeenCalled();
-    });
-
-    it('can update a single location', async () => {
-      const catalog = ({
-        addOrUpdateComponent: jest.fn(),
-        locations: jest.fn().mockResolvedValue([
-          {
-            id: '123',
-            type: 'some',
-            target: 'thing',
-          },
-        ]),
-      } as unknown) as Catalog;
-
-      const locationReader = jest.fn().mockResolvedValue([
-        {
-          name: 'c1',
-        },
-      ]);
-
-      await expect(
-        CatalogLogic.refreshLocations(catalog, locationReader, logger),
-      ).resolves.toBeUndefined();
-      expect(locationReader).toHaveBeenCalledTimes(1);
-      expect(locationReader).toHaveBeenNthCalledWith(
-        1,
-        expect.objectContaining({ type: 'some' }),
-      );
-      expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1);
-      expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith(
-        1,
-        '123',
-        expect.objectContaining({ name: 'c1' }),
-      );
-    });
-  });
-});

plugins/catalog-backend/src/catalog/CatalogLogic.ts

@@ -1,68 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Logger } from 'winston';
-import { LocationReader } from '../ingestion/types';
-import { Catalog } from './types';
-
-export class CatalogLogic {
-  public static startRefreshLoop(
-    catalog: Catalog,
-    reader: LocationReader,
-    logger: Logger,
-  ): () => void {
-    let cancel: () => void;
-    let cancelled = false;
-    const cancellationPromise = new Promise((resolve) => {
-      cancel = () => {
-        resolve();
-        cancelled = true;
-      };
-    });
-
-    const startRefresh = async () => {
-      while (!cancelled) {
-        await CatalogLogic.refreshLocations(catalog, reader, logger);
-        await Promise.race([
-          new Promise((resolve) => setTimeout(resolve, 10000)),
-          cancellationPromise,
-        ]);
-      }
-    };
-    startRefresh();
-
-    return cancel!;
-  }
-
-  public static async refreshLocations(
-    catalog: Catalog,
-    reader: LocationReader,
-    logger: Logger,
-  ): Promise<void> {
-    const locations = await catalog.locations();
-    for (const location of locations) {
-      try {
-        logger.debug(`Attempting refresh of location: ${location.id}`);
-        const componentDescriptors = await reader(location);
-        for (const componentDescriptor of componentDescriptors) {
-          await catalog.addOrUpdateComponent(location.id, componentDescriptor);
-        }
-      } catch (e) {
-        logger.debug(`Failed to update location "${location.id}", ${e}`);
-      }
-    }
-  }
-}

plugins/catalog-backend/src/catalog/DatabaseCatalog.ts

@@ -1,110 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { NotFoundError } from '@backstage/backend-common';
-import Knex from 'knex';
-import path from 'path';
-import { v4 as uuidv4 } from 'uuid';
-import { Logger } from 'winston';
-import { ComponentDescriptor } from '../descriptors';
-import { readLocation } from '../ingestion';
-import { CatalogLogic } from './CatalogLogic';
-import { AddLocationRequest, Catalog, Component, Location } from './types';
-
-export class DatabaseCatalog implements Catalog {
-  static async create(
-    database: Knex,
-    logger: Logger,
-  ): Promise<DatabaseCatalog> {
-    await database.migrate.latest({
-      directory: path.resolve(__dirname, '..', 'migrations'),
-      loadExtensions: ['.js'],
-    });
-
-    const databaseCatalog = new DatabaseCatalog(database, logger);
-    CatalogLogic.startRefreshLoop(databaseCatalog, readLocation, logger);
-
-    return databaseCatalog;
-  }
-
-  constructor(
-    private readonly database: Knex,
-    private readonly logger: Logger,
-  ) {}
-
-  async addOrUpdateComponent(
-    locationId: string,
-    descriptor: ComponentDescriptor,
-  ): Promise<void> {
-    const component: Component = {
-      name: descriptor.metadata.name,
-    };
-
-    await this.database.transaction(async (tx) => {
-      // TODO(freben): Currently, several locations can compete for the same component
-      const count = await tx('components')
-        .where({ name: component.name })
-        .update({ ...component, locationId });
-      if (!count) {
-        await tx('components').insert({
-          ...component,
-          id: uuidv4(),
-          locationId,
-        });
-      }
-    });
-  }
-
-  async components(): Promise<Component[]> {
-    return await this.database('components').select();
-  }
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async component(name: string): Promise<Component> {
-    const items = await this.database('components').where({ name }).select();
-    if (!items.length) {
-      throw new NotFoundError(`Found no component with name ${name}`);
-    }
-    return items[0];
-  }
-
-  async addLocation(location: AddLocationRequest): Promise<Location> {
-    const id = uuidv4();
-    const { type, target } = location;
-    await this.database('locations').insert({ id, type, target });
-    return await this.location(id);
-  }
-
-  async removeLocation(id: string): Promise<void> {
-    const result = await this.database('locations').where({ id }).del();
-
-    if (!result) {
-      throw new NotFoundError(`Found no location with ID ${id}`);
-    }
-  }
-
-  async location(id: string): Promise<Location> {
-    const items = await this.database('locations').where({ id }).select();
-    if (!items.length) {
-      throw new NotFoundError(`Found no location with ID ${id}`);
-    }
-    return items[0];
-  }
-
-  async locations(): Promise<Location[]> {
-    return await this.database('locations').select();
-  }
-}

plugins/catalog-backend/src/catalog/DatabaseItemsCatalog.ts

@@ -14,17 +14,19 @@
  * limitations under the License.
  */
 
-import { Location } from '../catalog/types';
-import { ComponentDescriptor } from '../descriptors';
-import { readFileLocation } from './fileLocation';
+import { Database } from '../database';
+import { Component, ItemsCatalog } from './types';
 
-export async function readLocation(
-  location: Location,
-): Promise<ComponentDescriptor[]> {
-  switch (location.type) {
-    case 'file':
-      return await readFileLocation(location.target);
-    default:
-      throw new Error(`Unknown type "${location.type}"`);
+export class DatabaseItemsCatalog implements ItemsCatalog {
+  constructor(private readonly database: Database) {}
+
+  async components(): Promise<Component[]> {
+    const items = await this.database.components();
+    return items;
+  }
+
+  async component(name: string): Promise<Component> {
+    const item = await this.database.component(name);
+    return item;
   }
 }

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Database } from '../database';
+import { AddLocation, Location, LocationsCatalog } from './types';
+
+export class DatabaseLocationsCatalog implements LocationsCatalog {
+  constructor(private readonly database: Database) {}
+
+  async addLocation(location: AddLocation): Promise<Location> {
+    const added = await this.database.addLocation(location);
+    return added;
+  }
+
+  async removeLocation(id: string): Promise<void> {
+    await this.database.removeLocation(id);
+  }
+
+  async locations(): Promise<Location[]> {
+    const items = await this.database.locations();
+    return items;
+  }
+
+  async location(id: string): Promise<Location> {
+    const item = await this.location(id);
+    return item;
+  }
+}

plugins/catalog-backend/src/catalog/StaticCatalog.ts

@@ -1,73 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { NotFoundError } from '@backstage/backend-common';
-import { v4 as uuidv4 } from 'uuid';
-import { ComponentDescriptor } from '../descriptors';
-import { AddLocationRequest, Catalog, Component, Location } from './types';
-
-export class StaticCatalog implements Catalog {
-  private _components: Component[];
-  private _locations: Location[];
-
-  constructor(components: Component[], locations: Location[]) {
-    this._components = components;
-    this._locations = locations;
-  }
-
-  async addOrUpdateComponent(
-    locationId: string,
-    descriptor: ComponentDescriptor,
-  ): Promise<void> {
-    this._components = this._components
-      .filter((c) => c.name !== descriptor.metadata.name)
-      .concat([{ name: descriptor.metadata.name }]);
-  }
-
-  async components(): Promise<Component[]> {
-    return this._components.slice();
-  }
-
-  async component(name: string): Promise<Component> {
-    const item = this._components.find((i) => i.name === name);
-    if (!item) {
-      throw new NotFoundError(`Found no component with name ${name}`);
-    }
-    return item;
-  }
-
-  async addLocation(location: AddLocationRequest): Promise<Location> {
-    const l = { id: uuidv4(), type: location.type, target: location.target };
-    this._locations.push(l);
-    return l;
-  }
-
-  async removeLocation(id: string): Promise<void> {
-    this._locations = this._locations.filter((l) => l.id !== id);
-  }
-
-  async location(id: string): Promise<Location> {
-    const location = this._locations.find((l) => l.id === id);
-    if (!location) {
-      throw new NotFoundError(`Found no location with ID ${id}`);
-    }
-    return location;
-  }
-
-  async locations(): Promise<Location[]> {
-    return this._locations;
-  }
-}

plugins/catalog-backend/src/catalog/StaticItemsCatalog.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { NotFoundError } from '@backstage/backend-common';
+import { Component, ItemsCatalog } from './types';
+
+export class StaticItemsCatalog implements ItemsCatalog {
+  private _components: Component[];
+
+  constructor(components: Component[]) {
+    this._components = components;
+  }
+
+  async components(): Promise<Component[]> {
+    return this._components.slice();
+  }
+
+  async component(name: string): Promise<Component> {
+    const item = this._components.find((i) => i.name === name);
+    if (!item) {
+      throw new NotFoundError(`Found no component with name ${name}`);
+    }
+    return item;
+  }
+}

plugins/catalog-backend/src/catalog/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
-export * from './DatabaseCatalog';
-export * from './StaticCatalog';
+export * from './DatabaseItemsCatalog';
+export * from './DatabaseLocationsCatalog';
+export * from './StaticItemsCatalog';
 export * from './types';

plugins/catalog-backend/src/catalog/types.ts

@@ -15,38 +15,46 @@
  */
 
 import * as yup from 'yup';
-import { ComponentDescriptor } from '../descriptors';
+
+//
+// Items
+//
 
 export type Component = {
+  id: string;
+  locationId?: string;
   name: string;
 };
 
+export type ItemsCatalog = {
+  components(): Promise<Component[]>;
+  component(id: string): Promise<Component>;
+};
+
+//
+// Locations
+//
+
 export type Location = {
   id: string;
   type: string;
   target: string;
 };
 
-export type AddLocationRequest = {
+export type AddLocation = {
   type: string;
   target: string;
 };
 
-export const addLocationRequestShape: yup.Schema<AddLocationRequest> = yup
+export const addLocationSchema: yup.Schema<AddLocation> = yup
   .object({
     type: yup.string().required(),
     target: yup.string().required(),
   })
   .noUnknown();
 
-export type Catalog = {
-  addOrUpdateComponent(
-    locationId: string,
-    descriptor: ComponentDescriptor,
-  ): Promise<void>;
-  components(): Promise<Component[]>;
-  component(id: string): Promise<Component>;
-  addLocation(location: AddLocationRequest): Promise<Location>;
+export type LocationsCatalog = {
+  addLocation(location: AddLocation): Promise<Location>;
   removeLocation(id: string): Promise<void>;
   locations(): Promise<Location[]>;
   location(id: string): Promise<Location>;

plugins/catalog-backend/src/database/Database.test.ts

@@ -16,11 +16,10 @@
 
 import knex from 'knex';
 import path from 'path';
-import { PassThrough } from 'stream';
-import winston from 'winston';
-import { DatabaseCatalog } from './DatabaseCatalog';
+import { Database } from './Database';
+import { AddDatabaseLocation, DatabaseLocation } from './types';
 
-describe('DatabaseCatalog', () => {
+describe('Database', () => {
   const database = knex({
     client: 'sqlite3',
     connection: ':memory:',
@@ -30,33 +29,33 @@ describe('DatabaseCatalog', () => {
     resource.run('PRAGMA foreign_keys = ON', () => {});
   });
 
-  const logger = winston.createLogger({
-    transports: [new winston.transports.Stream({ stream: new PassThrough() })],
-  });
-
   beforeEach(async () => {
     await database.migrate.latest({
-      directory: path.resolve(__dirname, '..', 'migrations'),
+      directory: path.resolve(__dirname, 'migrations'),
       loadExtensions: ['.ts'],
     });
   });
 
   it('manages locations', async () => {
-    const catalog = new DatabaseCatalog(database, logger);
-    const input = { type: 'a', target: 'b' };
-    const output = { id: expect.anything(), type: 'a', target: 'b' };
+    const db = new Database(database);
+    const input: AddDatabaseLocation = { type: 'a', target: 'b' };
+    const output: DatabaseLocation = {
+      id: expect.anything(),
+      type: 'a',
+      target: 'b',
+    };
 
-    await catalog.addLocation(input);
+    await db.addLocation(input);
 
-    const locations = await catalog.locations();
+    const locations = await db.locations();
     expect(locations).toEqual([output]);
-    const location = await catalog.location(locations[0].id);
+    const location = await db.location(locations[0].id);
     expect(location).toEqual(output);
 
-    await catalog.removeLocation(locations[0].id);
+    await db.removeLocation(locations[0].id);
 
-    await expect(catalog.locations()).resolves.toEqual([]);
-    await expect(catalog.location(locations[0].id)).rejects.toThrow(
+    await expect(db.locations()).resolves.toEqual([]);
+    await expect(db.location(locations[0].id)).rejects.toThrow(
       /Found no location/,
     );
   });

plugins/catalog-backend/src/database/Database.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { NotFoundError } from '@backstage/backend-common';
+import Knex from 'knex';
+import { v4 as uuidv4 } from 'uuid';
+import {
+  AddDatabaseComponent,
+  AddDatabaseLocation,
+  DatabaseComponent,
+  DatabaseLocation,
+} from './types';
+
+export class Database {
+  constructor(private readonly database: Knex) {}
+
+  async addOrUpdateComponent(component: AddDatabaseComponent): Promise<void> {
+    await this.database.transaction(async (tx) => {
+      // TODO(freben): Currently, several locations can compete for the same component
+      // TODO(freben): If locationId is unset in the input, it won't be overwritten - should we instead replace with null?
+      const count = await tx<DatabaseComponent>('components')
+        .where({ name: component.name })
+        .update({ ...component });
+      if (!count) {
+        await tx<DatabaseComponent>('components').insert({
+          ...component,
+          id: uuidv4(),
+        });
+      }
+    });
+  }
+
+  async components(): Promise<DatabaseComponent[]> {
+    return await this.database<DatabaseComponent>('components')
+      .orderBy('name')
+      .select();
+  }
+
+  async component(name: string): Promise<DatabaseComponent> {
+    const items = await this.database<DatabaseComponent>('components')
+      .where({ name })
+      .select();
+    if (!items.length) {
+      throw new NotFoundError(`Found no component with name ${name}`);
+    }
+    return items[0];
+  }
+
+  async addLocation(location: AddDatabaseLocation): Promise<DatabaseLocation> {
+    const id = uuidv4();
+    const { type, target } = location;
+    await this.database<DatabaseLocation>('locations').insert({
+      id,
+      type,
+      target,
+    });
+    return await this.location(id);
+  }
+
+  async removeLocation(id: string): Promise<void> {
+    const result = await this.database<DatabaseLocation>('locations')
+      .where({ id })
+      .del();
+
+    if (!result) {
+      throw new NotFoundError(`Found no location with ID ${id}`);
+    }
+  }
+
+  async location(id: string): Promise<DatabaseLocation> {
+    const items = await this.database<DatabaseLocation>('locations')
+      .where({ id })
+      .select();
+    if (!items.length) {
+      throw new NotFoundError(`Found no location with ID ${id}`);
+    }
+    return items[0];
+  }
+
+  async locations(): Promise<DatabaseLocation[]> {
+    return await this.database<DatabaseLocation>('locations').select();
+  }
+}

plugins/catalog-backend/src/database/DatabaseManager.test.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PassThrough } from 'stream';
+import winston from 'winston';
+import {
+  ComponentDescriptor,
+  DescriptorParser,
+  LocationReader,
+} from '../ingestion';
+import { Database } from './Database';
+import { DatabaseManager } from './DatabaseManager';
+import { DatabaseLocation } from './types';
+
+describe('DatabaseManager', () => {
+  const logger = winston.createLogger({
+    transports: [new winston.transports.Stream({ stream: new PassThrough() })],
+  });
+
+  describe('refreshLocations', () => {
+    it('works with no locations added', async () => {
+      const db = ({
+        addOrUpdateComponent: jest.fn(),
+        locations: jest.fn().mockResolvedValue([]),
+      } as unknown) as Database;
+      const reader: LocationReader = {
+        read: jest.fn(),
+      };
+      const parser: DescriptorParser = {
+        parse: jest.fn(),
+      };
+
+      await expect(
+        DatabaseManager.refreshLocations(db, reader, parser, logger),
+      ).resolves.toBeUndefined();
+      expect(reader.read).not.toHaveBeenCalled();
+      expect(parser.parse).not.toHaveBeenCalled();
+    });
+
+    it('can update a single location', async () => {
+      const db = ({
+        addOrUpdateComponent: jest.fn(),
+        locations: jest.fn(() =>
+          Promise.resolve([
+            {
+              id: '123',
+              type: 'some',
+              target: 'thing',
+            } as DatabaseLocation,
+          ]),
+        ),
+      } as unknown) as Database;
+
+      const desc: ComponentDescriptor = {
+        metadata: { name: 'c1' },
+        spec: { type: 'service' },
+      };
+      const reader: LocationReader = {
+        read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
+      };
+      const parser: DescriptorParser = {
+        parse: jest.fn(() =>
+          Promise.resolve({ kind: 'Component', component: desc }),
+        ),
+      };
+
+      await expect(
+        DatabaseManager.refreshLocations(db, reader, parser, logger),
+      ).resolves.toBeUndefined();
+      expect(reader.read).toHaveBeenCalledTimes(1);
+      expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing');
+      expect(db.addOrUpdateComponent).toHaveBeenCalledTimes(1);
+      expect(db.addOrUpdateComponent).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({ locationId: '123', name: 'c1' }),
+      );
+    });
+  });
+});

plugins/catalog-backend/src/database/DatabaseManager.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Knex from 'knex';
+import path from 'path';
+import { Logger } from 'winston';
+import { DescriptorParser, LocationReader } from '../ingestion';
+import { Database } from './Database';
+import { AddDatabaseComponent } from './types';
+
+export class DatabaseManager {
+  public static async createDatabase(database: Knex): Promise<Database> {
+    await database.migrate.latest({
+      directory: path.resolve(__dirname, 'migrations'),
+      loadExtensions: ['.js'],
+    });
+    return new Database(database);
+  }
+
+  public static async refreshLocations(
+    database: Database,
+    reader: LocationReader,
+    parser: DescriptorParser,
+    logger: Logger,
+  ): Promise<void> {
+    const locations = await database.locations();
+    for (const location of locations) {
+      try {
+        logger.debug(
+          `Refreshing location ${location.id} type "${location.type}" target "${location.target}"`,
+        );
+
+        const readerOutput = await reader.read(location.type, location.target);
+        for (const readerItem of readerOutput) {
+          if (readerItem.type === 'error') {
+            logger.debug(readerItem.error);
+            continue;
+          }
+
+          const parserOutput = await parser.parse(readerItem.data);
+          if (parserOutput.kind === 'Component') {
+            const component = parserOutput.component;
+            const dbc: AddDatabaseComponent = {
+              locationId: location.id,
+              name: component.metadata.name,
+            };
+            await database.addOrUpdateComponent(dbc);
+          }
+        }
+      } catch (e) {
+        // TODO(freben): Store trace log of these events, or at least the
+        // latest status, per location
+        logger.debug(`Failed to refresh location ${location.id}, ${e}`);
+      }
+    }
+  }
+}

plugins/catalog-backend/src/database/index.ts

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-describe('test', () => {
-  it('unbreaks the test runner', () => {
-    expect(true).toBeTruthy();
-  });
-});
+export * from './Database';
+export * from './DatabaseManager';
+export * from './types';

plugins/catalog-backend/src/database/types.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as yup from 'yup';
+
+export type DatabaseComponent = {
+  id: string;
+  locationId?: string;
+  name: string;
+};
+
+export type AddDatabaseComponent = {
+  locationId?: string;
+  name: string;
+};
+
+export const addDatabaseComponentSchema: yup.Schema<AddDatabaseComponent> = yup
+  .object({
+    locationId: yup.string().optional(),
+    name: yup.string().required(),
+  })
+  .noUnknown();
+
+export type DatabaseLocation = {
+  id: string;
+  type: string;
+  target: string;
+};
+
+export type AddDatabaseLocation = {
+  type: string;
+  target: string;
+};
+
+export const addDatabaseLocationSchema: yup.Schema<AddDatabaseLocation> = yup
+  .object({
+    type: yup.string().required(),
+    target: yup.string().required(),
+  })
+  .noUnknown();

plugins/catalog-backend/src/index.ts

@@ -15,4 +15,7 @@
  */
 
 export * from './catalog';
+export * from './database';
+export * from './ingestion';
 export * from './service/router';
+export * from './util';

plugins/catalog-backend/src/ingestion/DescriptorParsers.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ComponentDescriptorV1Parser } from './descriptors/ComponentDescriptorV1Parser';
+import { parseDescriptorEnvelope } from './descriptors/DescriptorEnvelope';
+import { EnvelopeParser } from './descriptors/types';
+import { DescriptorParser, ParserOutput } from './types';
+
+export class DescriptorParsers implements DescriptorParser {
+  static create(): DescriptorParser {
+    return new DescriptorParsers([new ComponentDescriptorV1Parser()]);
+  }
+
+  constructor(private readonly parsers: EnvelopeParser[]) {}
+
+  async parse(descriptor: object): Promise<ParserOutput> {
+    const envelope = await parseDescriptorEnvelope(descriptor);
+
+    for (const parser of this.parsers) {
+      const parsed = await parser.tryParse(envelope);
+      if (parsed) {
+        return parsed;
+      }
+    }
+
+    throw new Error(
+      `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`,
+    );
+  }
+}

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FileLocationSource } from './sources/FileLocationSource';
+import { LocationSource } from './sources/types';
+import { LocationReader, ReaderOutput } from './types';
+
+export class LocationReaders implements LocationReader {
+  static create(): LocationReader {
+    return new LocationReaders({
+      file: new FileLocationSource(),
+    });
+  }
+
+  constructor(private readonly sources: Record<string, LocationSource>) {}
+
+  async read(type: string, target: string): Promise<ReaderOutput[]> {
+    const source = this.sources[type];
+    if (!source) {
+      throw new Error(`Unknown location type ${type}`);
+    }
+
+    return source.read(target);
+  }
+}

plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1Parser.ts

@@ -15,9 +15,11 @@
  */
 
 import * as yup from 'yup';
-import { DescriptorEnvelope } from './envelope';
+import { ParserOutput } from '../types';
+import { DescriptorEnvelope } from './DescriptorEnvelope';
+import { EnvelopeParser } from './types';
 
-export type ComponentDescriptor = {
+export type ComponentDescriptorV1 = {
   metadata: {
     name: string;
   };
@@ -26,11 +28,7 @@ export type ComponentDescriptor = {
   };
 };
 
-const componentDescriptorSchema: yup.Schema<ComponentDescriptor> = yup.object({
-  kind: yup
-    .string()
-    .required()
-    .matches(/^Component$/),
+const schema: yup.Schema<ComponentDescriptorV1> = yup.object({
   metadata: yup.object({
     name: yup.string().required(),
   }),
@@ -39,17 +37,27 @@ const componentDescriptorSchema: yup.Schema<ComponentDescriptor> = yup.object({
   }),
 });
 
-export async function parseComponentDescriptor(
-  envelope: DescriptorEnvelope,
-): Promise<ComponentDescriptor[]> {
-  let componentDescriptor;
-  try {
-    componentDescriptor = await componentDescriptorSchema.validate(envelope, {
-      strict: true,
-    });
-  } catch (e) {
-    throw new Error(`Malformed component, ${e}`);
-  }
+export class ComponentDescriptorV1Parser implements EnvelopeParser {
+  async tryParse(
+    envelope: DescriptorEnvelope,
+  ): Promise<ParserOutput | undefined> {
+    if (
+      envelope.apiVersion !== 'catalog.backstage.io/v1' ||
+      envelope.kind !== 'Component'
+    ) {
+      return undefined;
+    }
 
-  return [componentDescriptor];
+    let component;
+    try {
+      component = await schema.validate(envelope, { strict: true });
+    } catch (e) {
+      throw new Error(`Malformed component, ${e}`);
+    }
+
+    return {
+      kind: 'Component',
+      component,
+    };
+  }
 }

plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelope.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import yaml from 'yaml';
 import * as yup from 'yup';
 
 export type DescriptorEnvelope = {
@@ -24,6 +23,7 @@ export type DescriptorEnvelope = {
   spec?: object;
 };
 
+// The schema of the envelope that's common to all versions/kinds
 const descriptorEnvelopeSchema: yup.Schema<DescriptorEnvelope> = yup
   .object({
     apiVersion: yup.string().required(),
@@ -33,20 +33,12 @@ const descriptorEnvelopeSchema: yup.Schema<DescriptorEnvelope> = yup
   })
   .noUnknown();
 
+// Validate some raw structured data as a descriptor envelope
 export async function parseDescriptorEnvelope(
-  rawYaml: string,
+  data: object,
 ): Promise<DescriptorEnvelope> {
-  let descriptor;
   try {
-    descriptor = yaml.parse(rawYaml);
-  } catch (e) {
-    throw new Error(`Malformed YAML, ${e}`);
-  }
-
-  try {
-    return await descriptorEnvelopeSchema.validate(descriptor, {
-      strict: true,
-    });
+    return await descriptorEnvelopeSchema.validate(data, { strict: true });
   } catch (e) {
     throw new Error(`Malformed envelope, ${e}`);
   }

plugins/catalog-backend/src/ingestion/descriptors/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ParserOutput } from '../types';
+import { DescriptorEnvelope } from './DescriptorEnvelope';
+
+export type EnvelopeParser = {
+  /**
+   * Parses and validates a single envelope into its materialized type.
+   *
+   * @param envelope A descriptor envelope
+   * @returns A materialized type, or undefined if the given version/kind is
+   *          not handled by this parser
+   * @throws An Error if the type was not properly formatted
+   */
+  tryParse(envelope: DescriptorEnvelope): Promise<ParserOutput | undefined>;
+};

plugins/catalog-backend/src/ingestion/index.ts

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 
-export * from './fileLocation';
+export * from './DescriptorParsers';
+export * from './LocationReaders';
 export * from './types';
-export * from './util';

plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts

@@ -15,21 +15,23 @@
  */
 
 import fs from 'fs-extra';
-import { ComponentDescriptor, parseDescriptor } from '../descriptors';
+import { ReaderOutput } from '../types';
+import { LocationSource } from './types';
+import { readDescriptorYaml } from './util';
 
-export async function readFileLocation(
-  target: string,
-): Promise<ComponentDescriptor[]> {
-  let rawYaml;
-  try {
-    rawYaml = await fs.readFile(target, 'utf8');
-  } catch (e) {
-    throw new Error(`Unable to read "${target}", ${e}`);
-  }
+export class FileLocationSource implements LocationSource {
+  async read(target: string): Promise<ReaderOutput[]> {
+    let rawYaml;
+    try {
+      rawYaml = await fs.readFile(target, 'utf8');
+    } catch (e) {
+      throw new Error(`Unable to read "${target}", ${e}`);
+    }
 
-  try {
-    return parseDescriptor(rawYaml);
-  } catch (e) {
-    throw new Error(`Malformed descriptor at "${target}", ${e}`);
+    try {
+      return readDescriptorYaml(rawYaml);
+    } catch (e) {
+      throw new Error(`Malformed descriptor at "${target}", ${e}`);
+    }
   }
 }

plugins/catalog-backend/src/ingestion/sources/types.ts

@@ -14,13 +14,15 @@
  * limitations under the License.
  */
 
-import { ComponentDescriptor, parseComponentDescriptor } from './component';
-import { parseDescriptorEnvelope } from './envelope';
+import { ReaderOutput } from '../types';
 
-// TODO(freben): Temporary helper that ignores the kind
-export async function parseDescriptor(
-  rawYaml: string,
-): Promise<ComponentDescriptor[]> {
-  const env = await parseDescriptorEnvelope(rawYaml);
-  return await parseComponentDescriptor(env);
-}
+export type LocationSource = {
+  /**
+   * Reads the contents of a single location.
+   *
+   * @param target The location target to read
+   * @returns The parsed contents, as an array of unverified descriptors
+   * @throws An error if the location target could not be read
+   */
+  read(target: string): Promise<ReaderOutput[]>;
+};

plugins/catalog-backend/src/ingestion/sources/util.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yaml from 'yaml';
+import { ReaderOutput } from '../types';
+
+export function readDescriptorYaml(data: string): ReaderOutput[] {
+  let documents;
+  try {
+    documents = yaml.parseAllDocuments(data);
+  } catch (e) {
+    throw new Error(`Could not parse YAML data, ${e}`);
+  }
+
+  const result: ReaderOutput[] = [];
+
+  for (const document of documents) {
+    if (document.contents) {
+      if (document.errors?.length) {
+        result.push({
+          type: 'error',
+          error: new Error(`Malformed YAML document, ${document.errors[0]}`),
+        });
+      } else {
+        const json = document.toJSON();
+        if (typeof json !== 'object' || Array.isArray(json)) {
+          result.push({
+            type: 'error',
+            error: new Error(`Malformed descriptor, expected object at root`),
+          });
+        } else {
+          result.push({
+            type: 'data',
+            data: json,
+          });
+        }
+      }
+    }
+  }
+
+  return result;
+}

plugins/catalog-backend/src/ingestion/types.ts

@@ -14,9 +14,39 @@
  * limitations under the License.
  */
 
-import { Location } from '../catalog';
-import { ComponentDescriptor } from '../descriptors';
+import { ComponentDescriptorV1 } from './descriptors/ComponentDescriptorV1Parser';
 
-export type LocationReader = (
-  location: Location,
-) => Promise<ComponentDescriptor[]>;
+export type ComponentDescriptor = ComponentDescriptorV1;
+
+export type ParserOutput = {
+  kind: 'Component';
+  component: ComponentDescriptor;
+};
+
+export type DescriptorParser = {
+  /**
+   * Parses and validates a single raw descriptor.
+   *
+   * @param descriptor A raw descriptor object
+   * @returns A structure describing the parsed and validated descriptor
+   * @throws An Error if the descriptor was malformed
+   */
+  parse(descriptor: object): Promise<ParserOutput>;
+};
+
+export type ReaderOutput =
+  | { type: 'error'; error: Error }
+  | { type: 'data'; data: object };
+
+export type LocationReader = {
+  /**
+   * Reads the contents of a single location.
+   *
+   * @param type The type of location to read
+   * @param target The location target (type-specific)
+   * @returns The parsed contents, as an array of unverified descriptors or
+   *          errors where the individual documents could not be parsed.
+   * @throws An error if the location as a whole could not be read
+   */
+  read(type: string, target: string): Promise<ReaderOutput[]>;
+};

plugins/catalog-backend/src/run.ts

@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-import yn from 'yn';
 import { getRootLogger } from '@backstage/backend-common';
+import yn from 'yn';
 import { startStandaloneServer } from './service/standaloneServer';
 
 const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 3003;
 const enableCors = yn(process.env.PLUGIN_CORS, { default: false });
 const logger = getRootLogger();
 
-startStandaloneServer({ port, enableCors, logger }).catch(err => {
+startStandaloneServer({ port, enableCors, logger }).catch((err) => {
   logger.error(err);
   process.exit(1);
 });

plugins/catalog-backend/src/service/router.ts

@@ -14,83 +14,62 @@
  * limitations under the License.
  */
 
-import { InputError } from '@backstage/backend-common';
-import express, { Request } from 'express';
+import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
-import yup from 'yup';
-import { addLocationRequestShape, Catalog } from '../catalog';
+import { addLocationSchema, ItemsCatalog, LocationsCatalog } from '../catalog';
+import { validateRequestBody } from './util';
 
 export interface RouterOptions {
-  catalog: Catalog;
+  itemsCatalog?: ItemsCatalog;
+  locationsCatalog?: LocationsCatalog;
   logger: Logger;
 }
 
-async function validateRequestBody<T>(
-  req: Request,
-  schema: yup.Schema<T>,
-): Promise<T> {
-  const contentType = req.header('content-type');
-  if (!contentType) {
-    throw new InputError('Content-Type missing');
-  } else if (!contentType.match(/^application\/json($|;)/)) {
-    throw new InputError('Illegal Content-Type');
-  }
-
-  const body = req.body;
-  if (!body) {
-    throw new InputError('Missing request body');
-  }
-
-  try {
-    await schema.validate(body, { strict: true });
-  } catch (e) {
-    throw new InputError(`Malformed request: ${e}`);
-  }
-
-  return body as T;
-}
-
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
-  const catalog = options.catalog;
+  const { itemsCatalog, locationsCatalog } = options;
   const logger = options.logger.child({ plugin: 'catalog' });
   const router = Router();
 
-  // Components
-  router
-    .get('/components', async (req, res) => {
-      const components = await catalog.components();
-      res.status(200).send(components);
-    })
-    .get('/components/:id', async (req, res) => {
-      const { id } = req.params;
-      const component = await catalog.component(id);
-      res.status(200).send(component);
-    });
+  if (itemsCatalog) {
+    // Components
+    router
+      .get('/components', async (req, res) => {
+        const components = await itemsCatalog.components();
+        res.status(200).send(components);
+      })
+      .get('/components/:id', async (req, res) => {
+        const { id } = req.params;
+        const component = await itemsCatalog.component(id);
+        res.status(200).send(component);
+      });
+  }
 
   // Locations
-  router
-    .post('/locations', async (req, res) => {
-      const input = await validateRequestBody(req, addLocationRequestShape);
-      const output = await catalog.addLocation(input);
-      res.status(201).send(output);
-    })
-    .get('/locations', async (req, res) => {
-      const output = await catalog.locations();
-      res.status(200).send(output);
-    })
-    .get('/locations/:id', async (req, res) => {
-      const { id } = req.params;
-      const output = await catalog.location(id);
-      res.status(200).send(output);
-    })
-    .delete('/locations/:id', async (req, res) => {
-      const { id } = req.params;
-      await catalog.removeLocation(id);
-      res.status(200).send();
-    });
+  if (locationsCatalog) {
+    router
+      .post('/locations', async (req, res) => {
+        const input = await validateRequestBody(req, addLocationSchema);
+        const output = await locationsCatalog.addLocation(input);
+        res.status(201).send(output);
+      })
+      .get('/locations', async (req, res) => {
+        const output = await locationsCatalog.locations();
+        res.status(200).send(output);
+      })
+      .get('/locations/:id', async (req, res) => {
+        const { id } = req.params;
+        const output = await locationsCatalog.location(id);
+        res.status(200).send(output);
+      })
+      .delete('/locations/:id', async (req, res) => {
+        const { id } = req.params;
+        await locationsCatalog.removeLocation(id);
+        res.status(200).send();
+      });
+  }
 
   const app = express();
   app.set('logger', logger);

plugins/catalog-backend/src/service/standaloneApplication.ts

@@ -24,19 +24,20 @@ import cors from 'cors';
 import express from 'express';
 import helmet from 'helmet';
 import { Logger } from 'winston';
-import { Catalog } from '../catalog';
+import { ItemsCatalog, LocationsCatalog } from '../catalog';
 import { createRouter } from './router';
 
 export interface ApplicationOptions {
   enableCors: boolean;
-  catalog: Catalog;
+  itemsCatalog: ItemsCatalog;
+  locationsCatalog?: LocationsCatalog;
   logger: Logger;
 }
 
 export async function createStandaloneApplication(
   options: ApplicationOptions,
 ): Promise<express.Application> {
-  const { enableCors, catalog, logger } = options;
+  const { enableCors, itemsCatalog, locationsCatalog, logger } = options;
   const app = express();
 
   app.use(helmet());
@@ -46,7 +47,7 @@ export async function createStandaloneApplication(
   app.use(compression());
   app.use(express.json());
   app.use(requestLoggingHandler());
-  app.use('/', await createRouter({ catalog, logger }));
+  app.use('/', await createRouter({ itemsCatalog, locationsCatalog, logger }));
   app.use(notFoundHandler());
   app.use(errorHandler());
 

plugins/catalog-backend/src/service/standaloneServer.ts

@@ -16,7 +16,7 @@
 
 import { Server } from 'http';
 import { Logger } from 'winston';
-import { StaticCatalog } from '../catalog';
+import { StaticItemsCatalog } from '../catalog';
 import { createStandaloneApplication } from './standaloneApplication';
 
 export interface ServerOptions {
@@ -30,20 +30,15 @@ export async function startStandaloneServer(
 ): Promise<Server> {
   const logger = options.logger.child({ service: 'catalog-backend' });
 
-  const catalog = new StaticCatalog(
-    [
-      { name: 'component1' },
-      { name: 'component2' },
-      { name: 'component3' },
-      { name: 'component4' },
-    ],
-    [],
-  );
+  const itemsCatalog = new StaticItemsCatalog([
+    { id: '1', name: 'component1' },
+    { id: '2', name: 'component2' },
+  ]);
 
   logger.debug('Creating application...');
   const app = await createStandaloneApplication({
     enableCors: options.enableCors,
-    catalog,
+    itemsCatalog,
     logger,
   });
 

plugins/catalog-backend/src/service/util.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { InputError } from '@backstage/backend-common';
+import { Request } from 'express';
+import yup from 'yup';
+
+export async function validateRequestBody<T>(
+  req: Request,
+  schema: yup.Schema<T>,
+): Promise<T> {
+  const contentType = req.header('content-type');
+  if (!contentType) {
+    throw new InputError('Content-Type missing');
+  } else if (!contentType.match(/^application\/json($|;)/)) {
+    throw new InputError('Illegal Content-Type');
+  }
+
+  const body = req.body;
+  if (!body) {
+    throw new InputError('Missing request body');
+  }
+
+  try {
+    await schema.validate(body, { strict: true });
+  } catch (e) {
+    throw new InputError(`Malformed request: ${e}`);
+  }
+
+  return body as T;
+}

plugins/catalog-backend/src/util/index.ts

@@ -14,6 +14,4 @@
  * limitations under the License.
  */
 
-export * from './component';
-export * from './envelope';
-export * from './util';
+export * from './runPeriodically';

plugins/catalog-backend/src/util/runPeriodically.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Runs a function repeatedly, with a fixed wait between invocations.
+ *
+ * Supports async functions, and silently ignores exceptions and rejections.
+ *
+ * @param fn The function to run. May return a Promise.
+ * @param delayMs The delay between a completed function invocation and the
+ *                next.
+ * @returns A function that, when called, stops the invocation loop.
+ */
+export function runPeriodically(fn: () => any, delayMs: number): () => void {
+  let cancel: () => void;
+  let cancelled = false;
+  const cancellationPromise = new Promise((resolve) => {
+    cancel = () => {
+      resolve();
+      cancelled = true;
+    };
+  });
+
+  const startRefresh = async () => {
+    while (!cancelled) {
+      try {
+        await fn();
+      } catch {
+        // ignore intentionally
+      }
+
+      await Promise.race([
+        new Promise((resolve) => setTimeout(resolve, delayMs)),
+        cancellationPromise,
+      ]);
+    }
+  };
+  startRefresh();
+
+  return cancel!;
+}