paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #940 from adaszyn/catalog-store-location-updates

Browse Source 
Store catalog location updates in database
This commit is contained in:
Wojciech Adaszyński
2020-05-20 17:31:43 +02:00
committed by GitHub
parent b29e03dfe8 b8999f48fe
commit e20f1d8867
8 changed files with 288 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/database/Database.ts
+20 -1
View File
@@ -22,6 +22,8 @@ import {
AddDatabaseLocation,
DatabaseComponent,
DatabaseLocation,
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
} from './types';
export class Database {
@@ -106,6 +108,23 @@ export class Database {
}
async locations(): Promise<DatabaseLocation[]> {
return await this.database<DatabaseLocation>('locations').select();
return this.database<DatabaseLocation>('locations').select();
}
async addLocationUpdateLogEvent(
locationId: string,
status: DatabaseLocationUpdateLogStatus,
componentName?: string,
message?: string,
): Promise<void> {
return this.database<DatabaseLocationUpdateLogEvent>(
'location_update_log',
).insert({
id: uuidv4(),
status: status,
location_id: locationId,
component_name: componentName,
message,
});
}
}
plugins/catalog-backend/src/database/DatabaseManager.test.ts
+141 -1
View File
@@ -20,10 +20,11 @@ import {
ComponentDescriptor,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { Database } from './Database';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocation } from './types';
import { DatabaseLocation, DatabaseLocationUpdateLogStatus } from './types';
describe('DatabaseManager', () => {
const logger = winston.createLogger({
@@ -62,6 +63,7 @@ describe('DatabaseManager', () => {
} as DatabaseLocation,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
@@ -90,5 +92,143 @@ describe('DatabaseManager', () => {
expect.objectContaining({ locationId: '123', name: 'c1' }),
);
});
it('logs successful updates', async () => {
const db = ({
addOrUpdateComponent: jest.fn(),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.resolve({ kind: 'Component', component: desc }),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, logger),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
'c1',
);
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
2,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
undefined,
);
});
it('logs unsuccessful updates when parser fails', async () => {
const db = ({
addOrUpdateComponent: jest.fn(),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, logger),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.FAIL,
'c1',
'parser error message',
);
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
2,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
undefined,
);
});
it('logs unsuccessful updates when reader fails', async () => {
const db = ({
addOrUpdateComponent: jest.fn(),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(() =>
Promise.reject([{ type: 'error', error: new Error('test message') }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, logger),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.FAIL,
undefined,
undefined,
);
});
});
});
plugins/catalog-backend/src/database/DatabaseManager.ts
+60 -15
View File
@@ -17,9 +17,9 @@
import Knex from 'knex';
import path from 'path';
import { Logger } from 'winston';
import { DescriptorParser, LocationReader } from '../ingestion';
import { DescriptorParser, LocationReader, ParserError } from '../ingestion';
import { Database } from './Database';
import { AddDatabaseComponent } from './types';
import { AddDatabaseComponent, DatabaseLocationUpdateLogStatus } from './types';
export class DatabaseManager {
public static async createDatabase(database: Knex): Promise<Database> {
@@ -30,6 +30,32 @@ export class DatabaseManager {
return new Database(database);
}
private static async logUpdateSuccess(
database: Database,
locationId: string,
componentName?: string,
) {
return database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.SUCCESS,
componentName,
);
}
private static async logUpdateFailure(
database: Database,
locationId: string,
error?: Error,
componentName?: string,
) {
return database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.FAIL,
componentName,
error?.message,
);
}
public static async refreshLocations(
database: Database,
reader: LocationReader,
@@ -44,26 +70,45 @@ export class DatabaseManager {
);
const readerOutput = await reader.read(location.type, location.target);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
logger.debug(readerItem.error);
continue;
}
const parserOutput = await parser.parse(readerItem.data);
if (parserOutput.kind === 'Component') {
const component = parserOutput.component;
const dbc: AddDatabaseComponent = {
locationId: location.id,
name: component.metadata.name,
};
await database.addOrUpdateComponent(dbc);
let parserOutput;
try {
parserOutput = await parser.parse(readerItem.data);
if (parserOutput.kind === 'Component') {
const component = parserOutput.component;
const dbc: AddDatabaseComponent = {
locationId: location.id,
name: component.metadata.name,
};
await database.addOrUpdateComponent(dbc);
await DatabaseManager.logUpdateSuccess(
database,
location.id,
component.metadata.name,
);
}
} catch (error) {
let componentName;
if (error instanceof ParserError) {
componentName = error.componentName;
}
await DatabaseManager.logUpdateFailure(
database,
location.id,
error,
componentName,
);
}
}
} catch (e) {
// TODO(freben): Store trace log of these events, or at least the
// latest status, per location
logger.debug(`Failed to refresh location ${location.id}, ${e}`);
await DatabaseManager.logUpdateSuccess(database, location.id);
} catch (error) {
logger.debug(`Failed to refresh location ${location.id}, ${error}`);
await DatabaseManager.logUpdateFailure(database, location.id, error);
}
}
}
plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as Knex from 'knex';
export async function up(knex: Knex): Promise<any> {
return knex.schema.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table.string('message');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.onUpdate('CASCADE')
.onDelete('CASCADE');
table.string('component_name').notNullable();
});
}
export async function down(knex: Knex): Promise<any> {
return knex.schema.dropTableIfExists('location_update_log');
}
plugins/catalog-backend/src/database/types.ts
+14
View File
@@ -51,3 +51,17 @@ export const addDatabaseLocationSchema: yup.Schema<AddDatabaseLocation> = yup
target: yup.string().required(),
})
.noUnknown();
export enum DatabaseLocationUpdateLogStatus {
FAIL = 'fail',
SUCCESS = 'success',
}
export type DatabaseLocationUpdateLogEvent = {
id: string;
status: DatabaseLocationUpdateLogStatus;
location_id: string;
component_name: string;
created_at?: string;
message?: string;
};
plugins/catalog-backend/src/ingestion/DescriptorParsers.ts
+3 -3
View File
@@ -17,7 +17,7 @@
import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser';
import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser';
import { KindParser } from './descriptors/types';
import { DescriptorParser, ParserOutput } from './types';
import { DescriptorParser, ParserError, ParserOutput } from './types';
import { makeValidator } from '../validation';
export class DescriptorParsers implements DescriptorParser {
@@ -41,9 +41,9 @@ export class DescriptorParsers implements DescriptorParser {
return parsed;
}
}
throw new Error(
throw new ParserError(
`Unsupported object ${envelope.apiVersion}, ${envelope.kind}`,
envelope.metadata?.name,
);
}
}
plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts
+5 -2
View File
@@ -15,7 +15,7 @@
*/
import * as yup from 'yup';
import { ParserOutput } from '../types';
import { ParserError, ParserOutput } from '../types';
import { DescriptorEnvelope } from './DescriptorEnvelopeParser';
import { KindParser } from './types';
@@ -62,7 +62,10 @@ export class ComponentDescriptorV1beta1Parser implements KindParser {
component: await this.schema.validate(envelope, { strict: true }),
};
} catch (e) {
throw new Error(`Malformed component, ${e}`);
throw new ParserError(
`Malformed component, ${e}`,
envelope.metadata?.name,
);
}
}
}
plugins/catalog-backend/src/ingestion/types.ts
+9
View File
@@ -34,6 +34,15 @@ export type DescriptorParser = {
parse(descriptor: object): Promise<ParserOutput>;
};
export class ParserError extends Error {
constructor(message?: string, private _componentName?: string | undefined) {
super(message);
}
get componentName() {
return this._componentName;
}
}
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: object };