Link components to the location that created them

packages/backend/src/index.ts

@@ -41,14 +41,16 @@ const DEFAULT_PORT = 7000;
 const PORT = parseInt(process.env.PORT ?? '', 10) || DEFAULT_PORT;
 
 function createEnv(plugin: string): PluginEnvironment {
-  return {
-    logger: getRootLogger().child({ type: 'plugin', plugin }),
-    database: knex({
-      client: 'sqlite3',
-      connection: ':memory:',
-      useNullAsDefault: true,
-    }),
-  };
+  const logger = getRootLogger().child({ type: 'plugin', plugin });
+  const database = knex({
+    client: 'sqlite3',
+    connection: ':memory:',
+    useNullAsDefault: true,
+  });
+  database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
+    resource.run('PRAGMA foreign_keys = ON', () => {});
+  });
+  return { logger, database };
 }
 
 async function main() {

plugins/catalog-backend/src/catalog/CatalogLogic.test.ts

@@ -67,6 +67,7 @@ describe('CatalogLogic', () => {
       expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1);
       expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith(
         1,
+        '123',
         expect.objectContaining({ name: 'c1' }),
       );
     });

plugins/catalog-backend/src/catalog/CatalogLogic.ts

@@ -56,9 +56,9 @@ export class CatalogLogic {
     for (const location of locations) {
       try {
         logger.debug(`Attempting refresh of location: ${location.id}`);
-        const components = await reader(location);
-        for (const component of components) {
-          await catalog.addOrUpdateComponent(component);
+        const componentDescriptors = await reader(location);
+        for (const componentDescriptor of componentDescriptors) {
+          await catalog.addOrUpdateComponent(location.id, componentDescriptor);
         }
       } catch (e) {
         logger.debug(`Failed to update location "${location.id}", ${e}`);

plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts

@@ -26,6 +26,9 @@ describe('DatabaseCatalog', () => {
     connection: ':memory:',
     useNullAsDefault: true,
   });
+  database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
+    resource.run('PRAGMA foreign_keys = ON', () => {});
+  });
 
   const logger = winston.createLogger({
     transports: [new winston.transports.Stream({ stream: new PassThrough() })],

plugins/catalog-backend/src/catalog/DatabaseCatalog.ts

@@ -19,6 +19,7 @@ import Knex from 'knex';
 import path from 'path';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
+import { ComponentDescriptor } from '../descriptors';
 import { readLocation } from '../ingestion';
 import { CatalogLogic } from './CatalogLogic';
 import { AddLocationRequest, Catalog, Component, Location } from './types';
@@ -44,15 +45,27 @@ export class DatabaseCatalog implements Catalog {
     private readonly logger: Logger,
   ) {}
 
-  async addOrUpdateComponent(component: Component): Promise<Component> {
+  async addOrUpdateComponent(
+    locationId: string,
+    descriptor: ComponentDescriptor,
+  ): Promise<void> {
+    const component: Component = {
+      name: descriptor.metadata.name,
+    };
+
     await this.database.transaction(async (tx) => {
-      await tx('components')
-        .insert(component)
-        .catch(() =>
-          tx('components').where({ name: component.name }).update(component),
-        );
+      // TODO(freben): Currently, several locations can compete for the same component
+      const count = await tx('components')
+        .where({ name: component.name })
+        .update({ ...component, locationId });
+      if (!count) {
+        await tx('components').insert({
+          ...component,
+          id: uuidv4(),
+          locationId,
+        });
+      }
     });
-    return component;
   }
 
   async components(): Promise<Component[]> {

plugins/catalog-backend/src/catalog/StaticCatalog.ts

@@ -16,6 +16,7 @@
 
 import { NotFoundError } from '@backstage/backend-common';
 import { v4 as uuidv4 } from 'uuid';
+import { ComponentDescriptor } from '../descriptors';
 import { AddLocationRequest, Catalog, Component, Location } from './types';
 
 export class StaticCatalog implements Catalog {
@@ -27,11 +28,13 @@ export class StaticCatalog implements Catalog {
     this._locations = locations;
   }
 
-  async addOrUpdateComponent(component: Component): Promise<Component> {
+  async addOrUpdateComponent(
+    locationId: string,
+    descriptor: ComponentDescriptor,
+  ): Promise<void> {
     this._components = this._components
-      .filter((c) => c.name !== component.name)
-      .concat([component]);
-    return component;
+      .filter((c) => c.name !== descriptor.metadata.name)
+      .concat([{ name: descriptor.metadata.name }]);
   }
 
   async components(): Promise<Component[]> {

plugins/catalog-backend/src/catalog/types.ts

@@ -15,6 +15,7 @@
  */
 
 import * as yup from 'yup';
+import { ComponentDescriptor } from '../descriptors';
 
 export type Component = {
   name: string;
@@ -39,7 +40,10 @@ export const addLocationRequestShape: yup.Schema<AddLocationRequest> = yup
   .noUnknown();
 
 export type Catalog = {
-  addOrUpdateComponent(component: Component): Promise<Component>;
+  addOrUpdateComponent(
+    locationId: string,
+    descriptor: ComponentDescriptor,
+  ): Promise<void>;
   components(): Promise<Component[]>;
   component(id: string): Promise<Component>;
   addLocation(location: AddLocationRequest): Promise<Location>;

plugins/catalog-backend/src/descriptors/component.ts

@@ -15,7 +15,6 @@
  */
 
 import * as yup from 'yup';
-import { Component } from '../catalog/types';
 import { DescriptorEnvelope } from './envelope';
 
 export type ComponentDescriptor = {
@@ -28,6 +27,10 @@ export type ComponentDescriptor = {
 };
 
 const componentDescriptorSchema: yup.Schema<ComponentDescriptor> = yup.object({
+  kind: yup
+    .string()
+    .required()
+    .matches(/^Component$/),
   metadata: yup.object({
     name: yup.string().required(),
   }),
@@ -38,7 +41,7 @@ const componentDescriptorSchema: yup.Schema<ComponentDescriptor> = yup.object({
 
 export async function parseComponentDescriptor(
   envelope: DescriptorEnvelope,
-): Promise<Component[]> {
+): Promise<ComponentDescriptor[]> {
   let componentDescriptor;
   try {
     componentDescriptor = await componentDescriptorSchema.validate(envelope, {
@@ -48,9 +51,5 @@ export async function parseComponentDescriptor(
     throw new Error(`Malformed component, ${e}`);
   }
 
-  const component: Component = {
-    name: componentDescriptor.metadata.name,
-  };
-
-  return [component];
+  return [componentDescriptor];
 }

plugins/catalog-backend/src/descriptors/util.ts

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-import { Component } from '../catalog/types';
-import { parseComponentDescriptor } from './component';
+import { ComponentDescriptor, parseComponentDescriptor } from './component';
 import { parseDescriptorEnvelope } from './envelope';
 
 // TODO(freben): Temporary helper that ignores the kind
-export async function parseDescriptor(rawYaml: string): Promise<Component[]> {
+export async function parseDescriptor(
+  rawYaml: string,
+): Promise<ComponentDescriptor[]> {
   const env = await parseDescriptorEnvelope(rawYaml);
   return await parseComponentDescriptor(env);
 }

plugins/catalog-backend/src/ingestion/fileLocation.ts

@@ -15,10 +15,11 @@
  */
 
 import fs from 'fs-extra';
-import { Component } from '../catalog/types';
-import { parseDescriptor } from '../descriptors';
+import { ComponentDescriptor, parseDescriptor } from '../descriptors';
 
-export async function readFileLocation(target: string): Promise<Component[]> {
+export async function readFileLocation(
+  target: string,
+): Promise<ComponentDescriptor[]> {
   let rawYaml;
   try {
     rawYaml = await fs.readFile(target, 'utf8');

plugins/catalog-backend/src/ingestion/types.ts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
-import { Component, Location } from '../catalog';
+import { Location } from '../catalog';
+import { ComponentDescriptor } from '../descriptors';
 
-export type LocationReader = (location: Location) => Promise<Component[]>;
+export type LocationReader = (
+  location: Location,
+) => Promise<ComponentDescriptor[]>;

plugins/catalog-backend/src/ingestion/util.ts

@@ -14,10 +14,13 @@
  * limitations under the License.
  */
 
-import { Component, Location } from '../catalog/types';
+import { Location } from '../catalog/types';
+import { ComponentDescriptor } from '../descriptors';
 import { readFileLocation } from './fileLocation';
 
-export async function readLocation(location: Location): Promise<Component[]> {
+export async function readLocation(
+  location: Location,
+): Promise<ComponentDescriptor[]> {
   switch (location.type) {
     case 'file':
       return await readFileLocation(location.target);

plugins/catalog-backend/src/migrations/20200511113813_init.ts

@@ -19,6 +19,9 @@ import * as Knex from 'knex';
 export async function up(knex: Knex): Promise<any> {
   return knex.schema
     .createTable('locations', (table) => {
+      table.comment(
+        'Registered locations that shall be contiuously scanned for catalog item updates',
+      );
       table.uuid('id').primary().comment('Auto-generated ID of the location');
       table.string('type').notNullable().comment('The type of location');
       table
@@ -27,7 +30,19 @@ export async function up(knex: Knex): Promise<any> {
         .comment('The actual target of the location');
     })
     .createTable('components', (table) => {
-      table.string('name').primary().comment('The name of the component');
+      table.comment('All components currently stored in the catalog');
+      table.uuid('id').primary().comment('Auto-generated ID of the component');
+      table
+        .uuid('locationId')
+        .references('id')
+        .inTable('locations')
+        .nullable()
+        .comment('The location that originated the component');
+      table
+        .string('name')
+        .unique()
+        .notNullable()
+        .comment('The external name of the component, as used in references');
     });
 }