diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8c685d3fb7..8a7a4c9b72 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -41,14 +41,16 @@ const DEFAULT_PORT = 7000; const PORT = parseInt(process.env.PORT ?? '', 10) || DEFAULT_PORT; function createEnv(plugin: string): PluginEnvironment { - return { - logger: getRootLogger().child({ type: 'plugin', plugin }), - database: knex({ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }), - }; + const logger = getRootLogger().child({ type: 'plugin', plugin }); + const database = knex({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + return { logger, database }; } async function main() { diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts index 38f1088b98..5234136b55 100644 --- a/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts +++ b/plugins/catalog-backend/src/catalog/CatalogLogic.test.ts @@ -67,6 +67,7 @@ describe('CatalogLogic', () => { expect(catalog.addOrUpdateComponent).toHaveBeenCalledTimes(1); expect(catalog.addOrUpdateComponent).toHaveBeenNthCalledWith( 1, + '123', expect.objectContaining({ name: 'c1' }), ); }); diff --git a/plugins/catalog-backend/src/catalog/CatalogLogic.ts b/plugins/catalog-backend/src/catalog/CatalogLogic.ts index ed195386d0..686aba0c40 100644 --- a/plugins/catalog-backend/src/catalog/CatalogLogic.ts +++ b/plugins/catalog-backend/src/catalog/CatalogLogic.ts @@ -56,9 +56,9 @@ export class CatalogLogic { for (const location of locations) { try { logger.debug(`Attempting refresh of location: ${location.id}`); - const components = await reader(location); - for (const component of components) { - await catalog.addOrUpdateComponent(component); + const componentDescriptors = await reader(location); + for (const componentDescriptor of componentDescriptors) { + await catalog.addOrUpdateComponent(location.id, componentDescriptor); } } catch (e) { logger.debug(`Failed to update location "${location.id}", ${e}`); diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts index 7a474246d9..98a5e9acd4 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.test.ts @@ -26,6 +26,9 @@ describe('DatabaseCatalog', () => { connection: ':memory:', useNullAsDefault: true, }); + database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); const logger = winston.createLogger({ transports: [new winston.transports.Stream({ stream: new PassThrough() })], diff --git a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts index 39a0719201..87ff5a503f 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseCatalog.ts @@ -19,6 +19,7 @@ import Knex from 'knex'; import path from 'path'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; +import { ComponentDescriptor } from '../descriptors'; import { readLocation } from '../ingestion'; import { CatalogLogic } from './CatalogLogic'; import { AddLocationRequest, Catalog, Component, Location } from './types'; @@ -44,15 +45,27 @@ export class DatabaseCatalog implements Catalog { private readonly logger: Logger, ) {} - async addOrUpdateComponent(component: Component): Promise { + async addOrUpdateComponent( + locationId: string, + descriptor: ComponentDescriptor, + ): Promise { + const component: Component = { + name: descriptor.metadata.name, + }; + await this.database.transaction(async (tx) => { - await tx('components') - .insert(component) - .catch(() => - tx('components').where({ name: component.name }).update(component), - ); + // TODO(freben): Currently, several locations can compete for the same component + const count = await tx('components') + .where({ name: component.name }) + .update({ ...component, locationId }); + if (!count) { + await tx('components').insert({ + ...component, + id: uuidv4(), + locationId, + }); + } }); - return component; } async components(): Promise { diff --git a/plugins/catalog-backend/src/catalog/StaticCatalog.ts b/plugins/catalog-backend/src/catalog/StaticCatalog.ts index b98798f644..28d6cff9a3 100644 --- a/plugins/catalog-backend/src/catalog/StaticCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticCatalog.ts @@ -16,6 +16,7 @@ import { NotFoundError } from '@backstage/backend-common'; import { v4 as uuidv4 } from 'uuid'; +import { ComponentDescriptor } from '../descriptors'; import { AddLocationRequest, Catalog, Component, Location } from './types'; export class StaticCatalog implements Catalog { @@ -27,11 +28,13 @@ export class StaticCatalog implements Catalog { this._locations = locations; } - async addOrUpdateComponent(component: Component): Promise { + async addOrUpdateComponent( + locationId: string, + descriptor: ComponentDescriptor, + ): Promise { this._components = this._components - .filter((c) => c.name !== component.name) - .concat([component]); - return component; + .filter((c) => c.name !== descriptor.metadata.name) + .concat([{ name: descriptor.metadata.name }]); } async components(): Promise { diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index ed33f35b3d..3a7b685003 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,6 +15,7 @@ */ import * as yup from 'yup'; +import { ComponentDescriptor } from '../descriptors'; export type Component = { name: string; @@ -39,7 +40,10 @@ export const addLocationRequestShape: yup.Schema = yup .noUnknown(); export type Catalog = { - addOrUpdateComponent(component: Component): Promise; + addOrUpdateComponent( + locationId: string, + descriptor: ComponentDescriptor, + ): Promise; components(): Promise; component(id: string): Promise; addLocation(location: AddLocationRequest): Promise; diff --git a/plugins/catalog-backend/src/descriptors/component.ts b/plugins/catalog-backend/src/descriptors/component.ts index 0ecaf77c66..abe75a0050 100644 --- a/plugins/catalog-backend/src/descriptors/component.ts +++ b/plugins/catalog-backend/src/descriptors/component.ts @@ -15,7 +15,6 @@ */ import * as yup from 'yup'; -import { Component } from '../catalog/types'; import { DescriptorEnvelope } from './envelope'; export type ComponentDescriptor = { @@ -28,6 +27,10 @@ export type ComponentDescriptor = { }; const componentDescriptorSchema: yup.Schema = yup.object({ + kind: yup + .string() + .required() + .matches(/^Component$/), metadata: yup.object({ name: yup.string().required(), }), @@ -38,7 +41,7 @@ const componentDescriptorSchema: yup.Schema = yup.object({ export async function parseComponentDescriptor( envelope: DescriptorEnvelope, -): Promise { +): Promise { let componentDescriptor; try { componentDescriptor = await componentDescriptorSchema.validate(envelope, { @@ -48,9 +51,5 @@ export async function parseComponentDescriptor( throw new Error(`Malformed component, ${e}`); } - const component: Component = { - name: componentDescriptor.metadata.name, - }; - - return [component]; + return [componentDescriptor]; } diff --git a/plugins/catalog-backend/src/descriptors/util.ts b/plugins/catalog-backend/src/descriptors/util.ts index e7b4944215..20118c7d97 100644 --- a/plugins/catalog-backend/src/descriptors/util.ts +++ b/plugins/catalog-backend/src/descriptors/util.ts @@ -14,12 +14,13 @@ * limitations under the License. */ -import { Component } from '../catalog/types'; -import { parseComponentDescriptor } from './component'; +import { ComponentDescriptor, parseComponentDescriptor } from './component'; import { parseDescriptorEnvelope } from './envelope'; // TODO(freben): Temporary helper that ignores the kind -export async function parseDescriptor(rawYaml: string): Promise { +export async function parseDescriptor( + rawYaml: string, +): Promise { const env = await parseDescriptorEnvelope(rawYaml); return await parseComponentDescriptor(env); } diff --git a/plugins/catalog-backend/src/ingestion/fileLocation.ts b/plugins/catalog-backend/src/ingestion/fileLocation.ts index 3c1b985356..e856e701d3 100644 --- a/plugins/catalog-backend/src/ingestion/fileLocation.ts +++ b/plugins/catalog-backend/src/ingestion/fileLocation.ts @@ -15,10 +15,11 @@ */ import fs from 'fs-extra'; -import { Component } from '../catalog/types'; -import { parseDescriptor } from '../descriptors'; +import { ComponentDescriptor, parseDescriptor } from '../descriptors'; -export async function readFileLocation(target: string): Promise { +export async function readFileLocation( + target: string, +): Promise { let rawYaml; try { rawYaml = await fs.readFile(target, 'utf8'); diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 5808c7e592..426188ed2c 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,6 +14,9 @@ * limitations under the License. */ -import { Component, Location } from '../catalog'; +import { Location } from '../catalog'; +import { ComponentDescriptor } from '../descriptors'; -export type LocationReader = (location: Location) => Promise; +export type LocationReader = ( + location: Location, +) => Promise; diff --git a/plugins/catalog-backend/src/ingestion/util.ts b/plugins/catalog-backend/src/ingestion/util.ts index 91b7ad2678..ede6b1b21a 100644 --- a/plugins/catalog-backend/src/ingestion/util.ts +++ b/plugins/catalog-backend/src/ingestion/util.ts @@ -14,10 +14,13 @@ * limitations under the License. */ -import { Component, Location } from '../catalog/types'; +import { Location } from '../catalog/types'; +import { ComponentDescriptor } from '../descriptors'; import { readFileLocation } from './fileLocation'; -export async function readLocation(location: Location): Promise { +export async function readLocation( + location: Location, +): Promise { switch (location.type) { case 'file': return await readFileLocation(location.target); diff --git a/plugins/catalog-backend/src/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/migrations/20200511113813_init.ts index 0db92a1f45..4c71fefddd 100644 --- a/plugins/catalog-backend/src/migrations/20200511113813_init.ts +++ b/plugins/catalog-backend/src/migrations/20200511113813_init.ts @@ -19,6 +19,9 @@ import * as Knex from 'knex'; export async function up(knex: Knex): Promise { return knex.schema .createTable('locations', (table) => { + table.comment( + 'Registered locations that shall be contiuously scanned for catalog item updates', + ); table.uuid('id').primary().comment('Auto-generated ID of the location'); table.string('type').notNullable().comment('The type of location'); table @@ -27,7 +30,19 @@ export async function up(knex: Knex): Promise { .comment('The actual target of the location'); }) .createTable('components', (table) => { - table.string('name').primary().comment('The name of the component'); + table.comment('All components currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the component'); + table + .uuid('locationId') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the component'); + table + .string('name') + .unique() + .notNullable() + .comment('The external name of the component, as used in references'); }); }