diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index d9278783ef..bdbf6571c7 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -22,6 +22,8 @@ import { AddDatabaseLocation, DatabaseComponent, DatabaseLocation, + DatabaseLocationUpdateLogEvent, + DatabaseLocationUpdateLogStatus, } from './types'; export class Database { @@ -106,6 +108,23 @@ export class Database { } async locations(): Promise { - return await this.database('locations').select(); + return this.database('locations').select(); + } + + async addLocationUpdateLogEvent( + locationId: string, + status: DatabaseLocationUpdateLogStatus, + componentName?: string, + message?: string, + ): Promise { + return this.database( + 'location_update_log', + ).insert({ + id: uuidv4(), + status: status, + location_id: locationId, + component_name: componentName, + message, + }); } } diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index 20b241555a..5d66b9edc0 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -20,10 +20,11 @@ import { ComponentDescriptor, DescriptorParser, LocationReader, + ParserError, } from '../ingestion'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; -import { DatabaseLocation } from './types'; +import { DatabaseLocation, DatabaseLocationUpdateLogStatus } from './types'; describe('DatabaseManager', () => { const logger = winston.createLogger({ @@ -62,6 +63,7 @@ describe('DatabaseManager', () => { } as DatabaseLocation, ]), ), + addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; const desc: ComponentDescriptor = { @@ -90,5 +92,143 @@ describe('DatabaseManager', () => { expect.objectContaining({ locationId: '123', name: 'c1' }), ); }); + + it('logs successful updates', async () => { + const db = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn(() => + Promise.resolve([ + { + id: '123', + type: 'some', + target: 'thing', + } as DatabaseLocation, + ]), + ), + addLocationUpdateLogEvent: jest.fn(), + } as unknown) as Database; + + const desc: ComponentDescriptor = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }; + const reader: LocationReader = { + read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + }; + const parser: DescriptorParser = { + parse: jest.fn(() => + Promise.resolve({ kind: 'Component', component: desc }), + ), + }; + + await expect( + DatabaseManager.refreshLocations(db, reader, parser, logger), + ).resolves.toBeUndefined(); + + expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( + 1, + '123', + DatabaseLocationUpdateLogStatus.SUCCESS, + 'c1', + ); + + expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( + 2, + '123', + DatabaseLocationUpdateLogStatus.SUCCESS, + undefined, + ); + }); + + it('logs unsuccessful updates when parser fails', async () => { + const db = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn(() => + Promise.resolve([ + { + id: '123', + type: 'some', + target: 'thing', + } as DatabaseLocation, + ]), + ), + addLocationUpdateLogEvent: jest.fn(), + } as unknown) as Database; + + const desc: ComponentDescriptor = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { name: 'c1' }, + spec: { type: 'service' }, + }; + const reader: LocationReader = { + read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + }; + const parser: DescriptorParser = { + parse: jest.fn(() => + Promise.reject(new ParserError('parser error message', 'c1')), + ), + }; + + await expect( + DatabaseManager.refreshLocations(db, reader, parser, logger), + ).resolves.toBeUndefined(); + + expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( + 1, + '123', + DatabaseLocationUpdateLogStatus.FAIL, + 'c1', + 'parser error message', + ); + + expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( + 2, + '123', + DatabaseLocationUpdateLogStatus.SUCCESS, + undefined, + ); + }); + + it('logs unsuccessful updates when reader fails', async () => { + const db = ({ + addOrUpdateComponent: jest.fn(), + locations: jest.fn(() => + Promise.resolve([ + { + id: '123', + type: 'some', + target: 'thing', + } as DatabaseLocation, + ]), + ), + addLocationUpdateLogEvent: jest.fn(), + } as unknown) as Database; + + const reader: LocationReader = { + read: jest.fn(() => + Promise.reject([{ type: 'error', error: new Error('test message') }]), + ), + }; + const parser: DescriptorParser = { + parse: jest.fn(() => + Promise.reject(new ParserError('parser error message', 'c1')), + ), + }; + + await expect( + DatabaseManager.refreshLocations(db, reader, parser, logger), + ).resolves.toBeUndefined(); + + expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( + 1, + '123', + DatabaseLocationUpdateLogStatus.FAIL, + undefined, + undefined, + ); + }); }); }); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 03ed066d49..d536d7e215 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -17,9 +17,9 @@ import Knex from 'knex'; import path from 'path'; import { Logger } from 'winston'; -import { DescriptorParser, LocationReader } from '../ingestion'; +import { DescriptorParser, LocationReader, ParserError } from '../ingestion'; import { Database } from './Database'; -import { AddDatabaseComponent } from './types'; +import { AddDatabaseComponent, DatabaseLocationUpdateLogStatus } from './types'; export class DatabaseManager { public static async createDatabase(database: Knex): Promise { @@ -30,6 +30,32 @@ export class DatabaseManager { return new Database(database); } + private static async logUpdateSuccess( + database: Database, + locationId: string, + componentName?: string, + ) { + return database.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.SUCCESS, + componentName, + ); + } + + private static async logUpdateFailure( + database: Database, + locationId: string, + error?: Error, + componentName?: string, + ) { + return database.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.FAIL, + componentName, + error?.message, + ); + } + public static async refreshLocations( database: Database, reader: LocationReader, @@ -44,26 +70,45 @@ export class DatabaseManager { ); const readerOutput = await reader.read(location.type, location.target); + for (const readerItem of readerOutput) { if (readerItem.type === 'error') { logger.debug(readerItem.error); continue; } - - const parserOutput = await parser.parse(readerItem.data); - if (parserOutput.kind === 'Component') { - const component = parserOutput.component; - const dbc: AddDatabaseComponent = { - locationId: location.id, - name: component.metadata.name, - }; - await database.addOrUpdateComponent(dbc); + let parserOutput; + try { + parserOutput = await parser.parse(readerItem.data); + if (parserOutput.kind === 'Component') { + const component = parserOutput.component; + const dbc: AddDatabaseComponent = { + locationId: location.id, + name: component.metadata.name, + }; + await database.addOrUpdateComponent(dbc); + await DatabaseManager.logUpdateSuccess( + database, + location.id, + component.metadata.name, + ); + } + } catch (error) { + let componentName; + if (error instanceof ParserError) { + componentName = error.componentName; + } + await DatabaseManager.logUpdateFailure( + database, + location.id, + error, + componentName, + ); } } - } catch (e) { - // TODO(freben): Store trace log of these events, or at least the - // latest status, per location - logger.debug(`Failed to refresh location ${location.id}, ${e}`); + await DatabaseManager.logUpdateSuccess(database, location.id); + } catch (error) { + logger.debug(`Failed to refresh location ${location.id}, ${error}`); + await DatabaseManager.logUpdateFailure(database, location.id, error); } } } diff --git a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts new file mode 100644 index 0000000000..c699ebb146 --- /dev/null +++ b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + return knex.schema.createTable('location_update_log', table => { + table.uuid('id').primary(); + table.enum('status', ['success', 'fail']).notNullable(); + table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table.string('message'); + table + .uuid('location_id') + .references('id') + .inTable('locations') + .onUpdate('CASCADE') + .onDelete('CASCADE'); + table.string('component_name').notNullable(); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.dropTableIfExists('location_update_log'); +} diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index ef7807b524..29ca248393 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -51,3 +51,17 @@ export const addDatabaseLocationSchema: yup.Schema = yup target: yup.string().required(), }) .noUnknown(); + +export enum DatabaseLocationUpdateLogStatus { + FAIL = 'fail', + SUCCESS = 'success', +} + +export type DatabaseLocationUpdateLogEvent = { + id: string; + status: DatabaseLocationUpdateLogStatus; + location_id: string; + component_name: string; + created_at?: string; + message?: string; +}; diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts index 151e2053e9..0b1860695a 100644 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts @@ -17,7 +17,7 @@ import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; import { KindParser } from './descriptors/types'; -import { DescriptorParser, ParserOutput } from './types'; +import { DescriptorParser, ParserError, ParserOutput } from './types'; import { makeValidator } from '../validation'; export class DescriptorParsers implements DescriptorParser { @@ -41,9 +41,9 @@ export class DescriptorParsers implements DescriptorParser { return parsed; } } - - throw new Error( + throw new ParserError( `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`, + envelope.metadata?.name, ); } } diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts index f7331f9838..9b62b89397 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts @@ -15,7 +15,7 @@ */ import * as yup from 'yup'; -import { ParserOutput } from '../types'; +import { ParserError, ParserOutput } from '../types'; import { DescriptorEnvelope } from './DescriptorEnvelopeParser'; import { KindParser } from './types'; @@ -62,7 +62,10 @@ export class ComponentDescriptorV1beta1Parser implements KindParser { component: await this.schema.validate(envelope, { strict: true }), }; } catch (e) { - throw new Error(`Malformed component, ${e}`); + throw new ParserError( + `Malformed component, ${e}`, + envelope.metadata?.name, + ); } } } diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index b6bd062383..7bef9d6912 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -34,6 +34,15 @@ export type DescriptorParser = { parse(descriptor: object): Promise; }; +export class ParserError extends Error { + constructor(message?: string, private _componentName?: string | undefined) { + super(message); + } + get componentName() { + return this._componentName; + } +} + export type ReaderOutput = | { type: 'error'; error: Error } | { type: 'data'; data: object };