Merge pull request #1890 from spotify/rugvip/static-loc

catalog-backend: add support for static location config + bootstrap location

packages/backend/src/plugins/catalog.ts

@@ -29,8 +29,9 @@ import { useHotCleanup } from '@backstage/backend-common';
 export default async function createPlugin({
   logger,
   database,
+  config,
 }: PluginEnvironment) {
-  const locationReader = new LocationReaders(logger);
+  const locationReader = new LocationReaders({ logger, config });
 
   const db = await DatabaseManager.createDatabase(database, { logger });
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);

plugins/catalog-backend/migrations/20200809202832_add_bootstrap_location.js

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+  // Adds a single 'bootstrap' location that can be used to trigger work in processors.
+  // This is primarily here to fulfill foreign key constraints.
+  await knex('locations').insert({
+    id: 'bootstrap',
+    type: 'bootstrap',
+    target: 'bootstrap',
+  });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+  await knex('locations')
+    .where({
+      id: 'bootstrap',
+    })
+    .del();
+};

plugins/catalog-backend/package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "@backstage/backend-common": "^0.1.1-alpha.18",
     "@backstage/catalog-model": "^0.1.1-alpha.18",
+    "@backstage/config": "^0.1.1-alpha.18",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts

@@ -17,6 +17,12 @@
 import { DatabaseManager } from '../database';
 import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
 
+const bootstrapLocation = {
+  id: 'bootstrap',
+  type: 'bootstrap',
+  target: 'bootstrap',
+};
+
 describe('DatabaseLocationsCatalog', () => {
   let catalog: DatabaseLocationsCatalog;
 
@@ -35,9 +41,12 @@ describe('DatabaseLocationsCatalog', () => {
     await expect(
       catalog.location('dd12620d-0436-422f-93bd-929aa0788123'),
     ).resolves.toEqual(expect.objectContaining({ data: location }));
-    await expect(catalog.locations()).resolves.toEqual([
-      expect.objectContaining({ data: location }),
-    ]);
+    await expect(catalog.locations()).resolves.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ data: location }),
+        expect.objectContaining({ data: bootstrapLocation }),
+      ]),
+    );
   });
 
   it('does not return duplicates of rows because of logs', async () => {
@@ -60,11 +69,12 @@ describe('DatabaseLocationsCatalog', () => {
       catalog.logUpdateSuccess(location1.id),
     ).resolves.toBeUndefined();
     const locations = await catalog.locations();
-    expect(locations.length).toBe(2);
+    expect(locations.length).toBe(3);
     expect(locations).toEqual(
       expect.arrayContaining([
         expect.objectContaining({ data: location1 }),
         expect.objectContaining({ data: location2 }),
+        expect.objectContaining({ data: bootstrapLocation }),
       ]),
     );
   });

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -24,6 +24,15 @@ import type {
   DbLocationsRowWithStatus,
 } from './types';
 
+const bootstrapLocation = {
+  id: 'bootstrap',
+  type: 'bootstrap',
+  target: 'bootstrap',
+  message: null,
+  status: null,
+  timestamp: null,
+};
+
 describe('CommonDatabase', () => {
   let db: Database;
   let entityRequest: DbEntityRequest;
@@ -85,8 +94,12 @@ describe('CommonDatabase', () => {
     await db.addLocation(input);
 
     const locations = await db.locations();
-    expect(locations).toEqual([output]);
-    const location = await db.location(locations[0].id);
+    expect(locations).toEqual(
+      expect.arrayContaining([output, bootstrapLocation]),
+    );
+    const location = await db.location(
+      locations.find(l => l.id !== 'bootstrap')!.id,
+    );
     expect(location).toEqual(output);
 
     // If we add 2 new update log events,
@@ -105,20 +118,21 @@ describe('CommonDatabase', () => {
       DatabaseLocationUpdateLogStatus.FAIL,
     );
 
-    expect(await db.locations()).toEqual([
-      {
-        ...output,
-        status: DatabaseLocationUpdateLogStatus.FAIL,
-        timestamp: expect.any(String),
-      },
-    ]);
-
-    await db.transaction(tx => db.removeLocation(tx, locations[0].id));
-
-    await expect(db.locations()).resolves.toEqual([]);
-    await expect(db.location(locations[0].id)).rejects.toThrow(
-      /Found no location/,
+    await expect(db.locations()).resolves.toEqual(
+      expect.arrayContaining([
+        bootstrapLocation,
+        {
+          ...output,
+          status: DatabaseLocationUpdateLogStatus.FAIL,
+          timestamp: expect.any(String),
+        },
+      ]),
     );
+
+    await db.transaction(tx => db.removeLocation(tx, location.id));
+
+    await expect(db.locations()).resolves.toEqual([bootstrapLocation]);
+    await expect(db.location(location.id)).rejects.toThrow(/Found no location/);
   });
 
   describe('addEntity', () => {

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -15,6 +15,7 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
+import { Config, ConfigReader } from '@backstage/config';
 import {
   Entity,
   EntityPolicies,
@@ -31,6 +32,7 @@ import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
 import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
 import { LocationRefProcessor } from './processors/LocationEntityProcessor';
+import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
 import * as result from './processors/results';
 import {
   LocationProcessor,
@@ -47,6 +49,12 @@ import { LocationReader, ReadLocationResult } from './types';
 // The max amount of nesting depth of generated work items
 const MAX_DEPTH = 10;
 
+type Options = {
+  logger?: Logger;
+  config?: Config;
+  processors?: LocationProcessor[];
+};
+
 /**
  * Implements the reading of a location through a series of processor tasks.
  */
@@ -54,10 +62,16 @@ export class LocationReaders implements LocationReader {
   private readonly logger: Logger;
   private readonly processors: LocationProcessor[];
 
-  static defaultProcessors(
-    entityPolicy: EntityPolicy = new EntityPolicies(),
-  ): LocationProcessor[] {
+  static defaultProcessors(options: {
+    config?: Config;
+    entityPolicy?: EntityPolicy;
+  }): LocationProcessor[] {
+    const {
+      config = new ConfigReader({}, 'missing-config'),
+      entityPolicy = new EntityPolicies(),
+    } = options;
     return [
+      StaticLocationProcessor.fromConfig(config),
       new FileReaderProcessor(),
       new GithubReaderProcessor(),
       new GithubApiReaderProcessor(),
@@ -71,10 +85,11 @@ export class LocationReaders implements LocationReader {
     ];
   }
 
-  constructor(
-    logger: Logger = getVoidLogger(),
-    processors: LocationProcessor[] = LocationReaders.defaultProcessors(),
-  ) {
+  constructor({
+    logger = getVoidLogger(),
+    config,
+    processors = LocationReaders.defaultProcessors({ config }),
+  }: Options) {
     this.logger = logger;
     this.processors = processors;
   }

plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import * as result from './results';
+import { Config } from '@backstage/config';
+import { LocationProcessorEmit } from './types';
+
+export class StaticLocationProcessor implements StaticLocationProcessor {
+  static fromConfig(config: Config): StaticLocationProcessor {
+    const locations: LocationSpec[] = [];
+
+    const lConfigs = config.getOptionalConfigArray('catalog.locations') ?? [];
+    for (const lConfig of lConfigs) {
+      const type = lConfig.getString('type');
+      const target = lConfig.getString('target');
+      locations.push({ type, target });
+    }
+
+    return new StaticLocationProcessor(locations);
+  }
+
+  constructor(private readonly staticLocations: LocationSpec[]) {}
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'bootstrap') {
+      return false;
+    }
+
+    for (const staticLocation of this.staticLocations) {
+      emit(result.location(staticLocation, false));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend/src/service/standaloneServer.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { createServiceBuilder } from '@backstage/backend-common';
+import {
+  createServiceBuilder,
+  loadBackendConfig,
+} from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 import { Server } from 'http';
 import { Logger } from 'winston';
 import { HigherOrderOperations } from '..';
@@ -34,12 +38,13 @@ export async function startStandaloneServer(
   options: ServerOptions,
 ): Promise<Server> {
   const logger = options.logger.child({ service: 'catalog-backend' });
+  const config = ConfigReader.fromConfigs(await loadBackendConfig());
 
   logger.debug('Creating application...');
   const db = await DatabaseManager.createInMemoryDatabase({ logger });
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);
   const locationsCatalog = new DatabaseLocationsCatalog(db);
-  const locationReader = new LocationReaders();
+  const locationReader = new LocationReaders({ logger, config });
   const higherOrderOperation = new HigherOrderOperations(
     entitiesCatalog,
     locationsCatalog,