From 83af809c9671828ce697d070e31595eaeb75be69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 22 May 2020 00:11:03 +0200 Subject: [PATCH] Store and retrieve the entire envelopes --- .../src/catalog/DatabaseEntitiesCatalog.ts | 11 +- .../src/catalog/StaticEntitiesCatalog.ts | 13 +- plugins/catalog-backend/src/catalog/types.ts | 11 +- .../src/database/Database.test.ts | 8 +- .../catalog-backend/src/database/Database.ts | 127 ++++++++++++++---- .../src/database/DatabaseManager.test.ts | 20 +-- .../src/database/DatabaseManager.ts | 7 +- .../migrations/20200511113813_init.ts | 81 +++++++---- plugins/catalog-backend/src/database/types.ts | 28 ++-- .../src/service/standaloneServer.ts | 14 +- 10 files changed, 215 insertions(+), 105 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index b829ab1aff..51d7ca98d7 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -15,18 +15,19 @@ */ import { Database } from '../database'; -import { EntitiesCatalog, Entity } from './types'; +import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; +import { EntitiesCatalog } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { constructor(private readonly database: Database) {} - async entities(): Promise { + async entities(): Promise { const items = await this.database.entities(); - return items; + return items.map(i => i.entity); } - async entity(name: string): Promise { + async entity(name: string): Promise { const item = await this.database.entity(name); - return item; + return item.entity; } } diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 4aedd94724..8c7ccb4bd1 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -15,21 +15,22 @@ */ import { NotFoundError } from '@backstage/backend-common'; -import { EntitiesCatalog, Entity } from './types'; +import { EntitiesCatalog } from './types'; +import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; export class StaticEntitiesCatalog implements EntitiesCatalog { - private _entities: Entity[]; + private _entities: DescriptorEnvelope[]; - constructor(entities: Entity[]) { + constructor(entities: DescriptorEnvelope[]) { this._entities = entities; } - async entities(): Promise { + async entities(): Promise { return this._entities.slice(); } - async entity(name: string): Promise { - const item = this._entities.find(e => e.name === name); + async entity(name: string): Promise { + const item = this._entities.find(e => e.metadata?.name === name); if (!item) { throw new NotFoundError(`Found no entity with name ${name}`); } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 898675c867..3c248f59e4 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,20 +15,15 @@ */ import * as yup from 'yup'; +import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; // // Items // -export type Entity = { - id: string; - locationId?: string; - name: string; -}; - export type EntitiesCatalog = { - entities(): Promise; - entity(id: string): Promise; + entities(): Promise; + entity(id: string): Promise; }; // diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index 5660d4ae35..459968412e 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -17,7 +17,7 @@ import knex from 'knex'; import path from 'path'; import { Database } from './Database'; -import { AddDatabaseLocation, DatabaseLocation } from './types'; +import { AddDatabaseLocation, DbLocationsRow } from './types'; describe('Database', () => { const database = knex({ @@ -39,7 +39,7 @@ describe('Database', () => { it('manages locations', async () => { const db = new Database(database); const input: AddDatabaseLocation = { type: 'a', target: 'b' }; - const output: DatabaseLocation = { + const output: DbLocationsRow = { id: expect.anything(), type: 'a', target: 'b', @@ -64,10 +64,10 @@ describe('Database', () => { // Prepare const catalog = new Database(database); const input: AddDatabaseLocation = { type: 'a', target: 'b' }; - const output1: DatabaseLocation = await catalog.addLocation(input); + const output1: DbLocationsRow = await catalog.addLocation(input); // Try to insert the same location - const output2: DatabaseLocation = await catalog.addLocation(input); + const output2: DbLocationsRow = await catalog.addLocation(input); const locations = await catalog.locations(); // Output is the same diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index c44af561cb..f0419df6d0 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -14,56 +14,131 @@ * limitations under the License. */ -import { NotFoundError } from '@backstage/backend-common'; +import { InputError, NotFoundError } from '@backstage/backend-common'; import Knex from 'knex'; import { v4 as uuidv4 } from 'uuid'; +import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; import { - AddDatabaseEntity, AddDatabaseLocation, - DatabaseEntity, - DatabaseLocation, DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, + DbEntitiesRow, + DbEntityRequest, + DbEntityResponse, + DbLocationsRow, } from './types'; +function serializeMetadata( + metadata: DescriptorEnvelope['metadata'], +): DbEntitiesRow['metadata'] { + if (!metadata) { + return null; + } + + const output = { ...metadata }; + // TODO: delete output.uid; + // TODO: delete output.generation; + + return JSON.stringify(output); +} + +function serializeSpec( + spec: DescriptorEnvelope['spec'], +): DbEntitiesRow['spec'] { + if (!spec) { + return null; + } + + return JSON.stringify(spec); +} + +function entityRequestToDb(request: DbEntityRequest): DbEntitiesRow { + return { + id: '', + location_id: request.locationId || null, + api_version: request.entity.apiVersion, + kind: request.entity.kind, + name: request.entity.metadata?.name || null, + namespace: request.entity.metadata?.namespace || null, + metadata: serializeMetadata(request.entity.metadata), + spec: serializeSpec(request.entity.spec), + }; +} + +function entityDbToResponse(row: DbEntitiesRow): DbEntityResponse { + const entity: DescriptorEnvelope = { + apiVersion: row.api_version, + kind: row.kind, + metadata: { + // TODO: uid: row.id, + // TODO: generation: row.generation, + }, + }; + + if (row.metadata) { + const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata']; + entity.metadata = { ...entity.metadata, ...metadata }; + } + + if (row.spec) { + const spec = JSON.parse(row.spec); + entity.spec = spec; + } + + return { + locationId: row.location_id || undefined, + entity, + }; +} + export class Database { constructor(private readonly database: Knex) {} - async addOrUpdateEntity(entity: AddDatabaseEntity): Promise { + async addOrUpdateEntity(request: DbEntityRequest): Promise { + if (!request.entity.metadata?.name) { + throw new InputError(`Entities without names are not yet supported`); + } + + const newRow = entityRequestToDb(request); + await this.database.transaction(async tx => { // TODO(freben): Currently, several locations can compete for the same entity // TODO(freben): If locationId is unset in the input, it won't be overwritten - should we instead replace with null? - const count = await tx('entities') - .where({ name: entity.name }) - .update({ ...entity }); + const count = await tx('entities') + .where({ name: request.entity.metadata?.name }) + .update({ + ...newRow, + id: undefined, + }); if (!count) { - await tx('entities').insert({ - ...entity, + await tx('entities').insert({ + ...newRow, id: uuidv4(), }); } }); } - async entities(): Promise { - return await this.database('entities') + async entities(): Promise { + const items = await this.database('entities') .orderBy('name') .select(); + return items.map(entityDbToResponse); } - async entity(name: string): Promise { - const items = await this.database('entities') + async entity(name: string): Promise { + const items = await this.database('entities') .where({ name }) .select(); if (!items.length) { throw new NotFoundError(`Found no entity with name ${name}`); } - return items[0]; + return entityDbToResponse(items[0]); } - async addLocation(location: AddDatabaseLocation): Promise { - return await this.database.transaction(async tx => { - const existingLocation = await tx('locations') + async addLocation(location: AddDatabaseLocation): Promise { + return await this.database.transaction(async tx => { + const existingLocation = await tx('locations') .where({ target: location.target, }) @@ -75,20 +150,18 @@ export class Database { const id = uuidv4(); const { type, target } = location; - await tx('locations').insert({ + await tx('locations').insert({ id, type, target, }); - return (await tx('locations') - .where({ id }) - .select())![0]; + return (await tx('locations').where({ id }).select())![0]; }); } async removeLocation(id: string): Promise { - const result = await this.database('locations') + const result = await this.database('locations') .where({ id }) .del(); @@ -97,8 +170,8 @@ export class Database { } } - async location(id: string): Promise { - const items = await this.database('locations') + async location(id: string): Promise { + const items = await this.database('locations') .where({ id }) .select(); if (!items.length) { @@ -107,8 +180,8 @@ export class Database { return items[0]; } - async locations(): Promise { - return this.database('locations').select(); + async locations(): Promise { + return this.database('locations').select(); } async addLocationUpdateLogEvent( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index 350d68bc14..1043733206 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -24,7 +24,7 @@ import { } from '../ingestion'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; -import { DatabaseLocation, DatabaseLocationUpdateLogStatus } from './types'; +import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; describe('DatabaseManager', () => { const logger = winston.createLogger({ @@ -60,7 +60,7 @@ describe('DatabaseManager', () => { id: '123', type: 'some', target: 'thing', - } as DatabaseLocation, + } as DbLocationsRow, ]), ), addLocationUpdateLogEvent: jest.fn(), @@ -85,10 +85,12 @@ describe('DatabaseManager', () => { expect(reader.read).toHaveBeenCalledTimes(1); expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing'); expect(db.addOrUpdateEntity).toHaveBeenCalledTimes(1); - expect(db.addOrUpdateEntity).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ locationId: '123', name: 'c1' }), - ); + expect(db.addOrUpdateEntity).toHaveBeenNthCalledWith(1, { + locationId: '123', + entity: expect.objectContaining({ + metadata: expect.objectContaining({ name: 'c1' }), + }), + }); }); it('logs successful updates', async () => { @@ -100,7 +102,7 @@ describe('DatabaseManager', () => { id: '123', type: 'some', target: 'thing', - } as DatabaseLocation, + } as DbLocationsRow, ]), ), addLocationUpdateLogEvent: jest.fn(), @@ -147,7 +149,7 @@ describe('DatabaseManager', () => { id: '123', type: 'some', target: 'thing', - } as DatabaseLocation, + } as DbLocationsRow, ]), ), addLocationUpdateLogEvent: jest.fn(), @@ -197,7 +199,7 @@ describe('DatabaseManager', () => { id: '123', type: 'some', target: 'thing', - } as DatabaseLocation, + } as DbLocationsRow, ]), ), addLocationUpdateLogEvent: jest.fn(), diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 251d098579..29b0badb7a 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -19,7 +19,7 @@ import path from 'path'; import { Logger } from 'winston'; import { DescriptorParser, LocationReader, ParserError } from '../ingestion'; import { Database } from './Database'; -import { AddDatabaseEntity, DatabaseLocationUpdateLogStatus } from './types'; +import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types'; export class DatabaseManager { public static async createDatabase(database: Knex): Promise { @@ -78,10 +78,7 @@ export class DatabaseManager { } try { const entity = await parser.parse(readerItem.data); - const dbc: AddDatabaseEntity = { - locationId: location.id, - name: entity.metadata!.name!, - }; + const dbc: DbEntityRequest = { locationId: location.id, entity }; await database.addOrUpdateEntity(dbc); await DatabaseManager.logUpdateSuccess( database, diff --git a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts index 8d3b0857ef..e31834d791 100644 --- a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts +++ b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts @@ -17,33 +17,60 @@ import * as Knex from 'knex'; export async function up(knex: Knex): Promise { - return knex.schema - .createTable('locations', table => { - table.comment( - 'Registered locations that shall be contiuously scanned for catalog item updates', - ); - table.uuid('id').primary().comment('Auto-generated ID of the location'); - table.string('type').notNullable().comment('The type of location'); - table - .string('target') - .notNullable() - .comment('The actual target of the location'); - }) - .createTable('entities', table => { - table.comment('All entities currently stored in the catalog'); - table.uuid('id').primary().comment('Auto-generated ID of the entity'); - table - .uuid('locationId') - .references('id') - .inTable('locations') - .nullable() - .comment('The location that originated the entity'); - table - .string('name') - .unique() - .notNullable() - .comment('The external name of the entity, as used in references'); - }); + return ( + knex.schema + // + // locations + // + .createTable('locations', table => { + table.comment( + 'Registered locations that shall be contiuously scanned for catalog item updates', + ); + table.uuid('id').primary().comment('Auto-generated ID of the location'); + table.string('type').notNullable().comment('The type of location'); + table + .string('target') + .notNullable() + .comment('The actual target of the location'); + }) + // + // entities + // + .createTable('entities', table => { + table.comment('All entities currently stored in the catalog'); + table.uuid('id').primary().comment('Auto-generated ID of the entity'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .nullable() + .comment('The location that originated the entity'); + table + .string('api_version') + .notNullable() + .comment('The apiVersion field of the entity'); + table + .string('kind') + .notNullable() + .comment('The kind field of the entity'); + table + .string('name') + .nullable() + .comment('The metadata.name field of the entity'); + table + .string('namespace') + .nullable() + .comment('The metadata.namespace field of the entity'); + table + .string('metadata') + .nullable() + .comment('The entire metadata JSON blob of the entity'); + table + .string('spec') + .nullable() + .comment('The entire spec JSON blob of the entity'); + }) + ); } export async function down(knex: Knex): Promise { diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 4ae87a8b60..33221a3fa6 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -15,26 +15,30 @@ */ import * as yup from 'yup'; +import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser'; -export type DatabaseEntity = { +export type DbEntitiesRow = { id: string; - locationId?: string; - name: string; + location_id: string | null; + api_version: string; + kind: string; + name: string | null; + namespace: string | null; + metadata: string | null; + spec: string | null; }; -export type AddDatabaseEntity = { +export type DbEntityRequest = { locationId?: string; - name: string; + entity: DescriptorEnvelope; }; -export const addDatabaseEntitySchema: yup.Schema = yup - .object({ - locationId: yup.string().optional(), - name: yup.string().required(), - }) - .noUnknown(); +export type DbEntityResponse = { + locationId?: string; + entity: DescriptorEnvelope; +}; -export type DatabaseLocation = { +export type DbLocationsRow = { id: string; type: string; target: string; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index f18ae37ea6..2fd4afc7cd 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -31,8 +31,18 @@ export async function startStandaloneServer( const logger = options.logger.child({ service: 'catalog-backend' }); const entitiesCatalog = new StaticEntitiesCatalog([ - { id: '1', name: 'entity1' }, - { id: '2', name: 'entity2' }, + { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }, + { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c2' }, + spec: { type: 'service' }, + }, ]); logger.debug('Creating application...');