Merge pull request #829 from spotify/mob/refresh

Implement first-light location refresh loop

packages/backend/src/plugins/catalog.ts

@@ -20,7 +20,7 @@ import {
 } from '@backstage/plugin-catalog-backend';
 import { PluginEnvironment } from '../types';
 
-export default async function ({ logger, database }: PluginEnvironment) {
-  const catalog = await DatabaseCatalog.create(database);
+export default async function({ logger, database }: PluginEnvironment) {
+  const catalog = await DatabaseCatalog.create(database, logger);
   return await createRouter({ catalog, logger });
 }

plugins/catalog-backend/fixtures/component1.yaml

@@ -0,0 +1 @@
+name: Component1

plugins/catalog-backend/package.json

@@ -17,11 +17,13 @@
     "cors": "^2.8.5",
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
+    "fs-extra": "^9.0.0",
     "helmet": "^3.22.0",
     "knex": "^0.21.1",
     "morgan": "^1.10.0",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",
+    "yaml": "^1.9.2",
     "yup": "^0.28.5"
   },
   "devDependencies": {

plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import knex from 'knex';
+import path from 'path';
+import { PassThrough } from 'stream';
+import winston from 'winston';
+import { DatabaseCatalog } from './DatabaseCatalog';
+
+describe('DatabaseCatalog', () => {
+  const database = knex({
+    client: 'sqlite3',
+    connection: ':memory:',
+    useNullAsDefault: true,
+  });
+
+  const logger = winston.createLogger({
+    transports: [new winston.transports.Stream({ stream: new PassThrough() })],
+  });
+
+  beforeEach(async () => {
+    await database.migrate.latest({
+      directory: path.resolve(__dirname, '..', 'migrations'),
+      loadExtensions: ['.ts'],
+    });
+  });
+
+  it('instantiates', () => {
+    const catalog = new DatabaseCatalog(database, logger);
+    expect(catalog).toBeDefined();
+  });
+
+  describe(`refreshLocations`, () => {
+    it('works with no locations added', async () => {
+      const catalog = new DatabaseCatalog(database, logger);
+      await expect(catalog.refreshLocations()).resolves.toBeUndefined();
+    });
+  });
+});

plugins/catalog-backend/src/catalog/DatabaseCatalog.ts

@@ -15,29 +15,79 @@
  */
 
 import { NotFoundError } from '@backstage/backend-common';
-import path from 'path';
 import Knex from 'knex';
+import path from 'path';
+import fs from 'fs-extra';
 import { v4 as uuidv4 } from 'uuid';
-import { AddLocationRequest, Component, Catalog, Location } from './types';
+import { Logger } from 'winston';
+import {
+  AddLocationRequest,
+  Catalog,
+  Component,
+  Location,
+  componentShape,
+} from './types';
+import YAML from 'yaml';
+
+async function readLocation(location: Location): Promise<Component[]> {
+  switch (location.type) {
+    case 'file': {
+      let parsed;
+      try {
+        const raw = await fs.readFile(location.target, 'utf8');
+        parsed = YAML.parse(raw);
+      } catch (e) {
+        throw new Error(`Unable to read "${location.target}", ${e}`);
+      }
+
+      try {
+        return [await componentShape.validate(parsed, { strict: true })];
+      } catch (e) {
+        throw new Error(
+          `Malformed file contents at "${location.target}", ${e}`,
+        );
+      }
+    }
+
+    default:
+      throw new Error(`Unknown type "${location.type}"`);
+  }
+}
 
 export class DatabaseCatalog implements Catalog {
-  static async create(database: Knex): Promise<DatabaseCatalog> {
+  static async create(
+    database: Knex,
+    logger: Logger,
+  ): Promise<DatabaseCatalog> {
     await database.migrate.latest({
       directory: path.resolve(__dirname, '..', 'migrations'),
       loadExtensions: ['.js'],
     });
 
-    return new DatabaseCatalog(database);
+    const databaseCatalog = new DatabaseCatalog(database, logger);
+
+    const startRefresh = async () => {
+      for (;;) {
+        await databaseCatalog.refreshLocations();
+        await new Promise(r => setTimeout(r, 10000));
+      }
+    };
+    startRefresh();
+
+    return databaseCatalog;
   }
 
-  constructor(private readonly database: Knex) {}
+  constructor(
+    private readonly database: Knex,
+    private readonly logger: Logger,
+  ) {}
 
   async components(): Promise<Component[]> {
     throw new Error('Not supported');
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async component(id: string): Promise<Component> {
+  async component(name: string): Promise<Component> {
     throw new Error('Not supported');
   }
 
@@ -49,15 +99,42 @@ export class DatabaseCatalog implements Catalog {
   }
 
   async removeLocation(id: string): Promise<void> {
-    const result = await this.database('locations').where({ id }).del();
+    const result = await this.database('locations')
+      .where({ id })
+      .del();
 
     if (!result) {
       throw new NotFoundError(`Found no location with ID ${id}`);
     }
   }
 
+  async refreshLocations(): Promise<void> {
+    const locations = await this.locations();
+    for (const location of locations) {
+      try {
+        this.logger.debug(`Attempting refresh of location: ${location.id}`);
+        const components = await readLocation(location);
+        for (const component of components) {
+          await this.database.transaction(async tx => {
+            await tx('components')
+              .insert(component)
+              .catch(() =>
+                tx('components')
+                  .where({ name: component.name })
+                  .update(component),
+              );
+          });
+        }
+      } catch (e) {
+        this.logger.debug(`Failed to update location "${location.id}", ${e}`);
+      }
+    }
+  }
+
   async location(id: string): Promise<Location> {
-    const items = await this.database('locations').where({ id }).select();
+    const items = await this.database('locations')
+      .where({ id })
+      .select();
     if (!items.length) {
       throw new NotFoundError(`Found no location with ID ${id}`);
     }

plugins/catalog-backend/src/catalog/StaticCatalog.ts

@@ -31,10 +31,10 @@ export class StaticCatalog implements Catalog {
     return this._components.slice();
   }
 
-  async component(id: string): Promise<Component> {
-    const item = this._components.find((i) => i.id === id);
+  async component(name: string): Promise<Component> {
+    const item = this._components.find(i => i.name === name);
     if (!item) {
-      throw new NotFoundError(`Found no component with ID ${id}`);
+      throw new NotFoundError(`Found no component with name ${name}`);
     }
     return item;
   }
@@ -46,11 +46,11 @@ export class StaticCatalog implements Catalog {
   }
 
   async removeLocation(id: string): Promise<void> {
-    this._locations = this._locations.filter((l) => l.id !== id);
+    this._locations = this._locations.filter(l => l.id !== id);
   }
 
   async location(id: string): Promise<Location> {
-    const location = this._locations.find((l) => l.id === id);
+    const location = this._locations.find(l => l.id === id);
     if (!location) {
       throw new NotFoundError(`Found no location with ID ${id}`);
     }

plugins/catalog-backend/src/catalog/types.ts

@@ -17,9 +17,15 @@
 import * as yup from 'yup';
 
 export type Component = {
-  id: string;
+  name: string;
 };
 
+export const componentShape: yup.Schema<Component> = yup
+  .object({
+    name: yup.string().required(),
+  })
+  .unknown();
+
 export type Location = {
   id: string;
   type: string;

plugins/catalog-backend/src/service/standaloneServer.ts

@@ -32,10 +32,10 @@ export async function startStandaloneServer(
 
   const catalog = new StaticCatalog(
     [
-      { id: 'component1' },
-      { id: 'component2' },
-      { id: 'component3' },
-      { id: 'component4' },
+      { name: 'component1' },
+      { name: 'component2' },
+      { name: 'component3' },
+      { name: 'component4' },
     ],
     [],
   );

yarn.lock

@@ -22136,6 +22136,13 @@ yaml@^1.7.2:
   dependencies:
     "@babel/runtime" "^7.8.7"
 
+yaml@^1.9.2:
+  version "1.9.2"
+  resolved "https://registry.npmjs.org/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed"
+  integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg==
+  dependencies:
+    "@babel/runtime" "^7.9.2"
+
 yargs-parser@^10.0.0:
   version "10.1.0"
   resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"