paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Merge pull request #958 from spotify/freben/store-and-retrieve

Browse Source 
Store and retrieve the entire envelopes
This commit is contained in:
Fredrik Adelöw
2020-05-22 13:54:19 +02:00
committed by GitHub
parent d68a621f51 83af809c96
commit 99a360bdef
10 changed files with 215 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+6 -5
View File
@@ -15,18 +15,19 @@
*/
import { Database } from '../database';
import { EntitiesCatalog, Entity } from './types';
import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
import { EntitiesCatalog } from './types';
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
constructor(private readonly database: Database) {}
async entities(): Promise<Entity[]> {
async entities(): Promise<DescriptorEnvelope[]> {
const items = await this.database.entities();
return items;
return items.map(i => i.entity);
}
async entity(name: string): Promise<Entity> {
async entity(name: string): Promise<DescriptorEnvelope> {
const item = await this.database.entity(name);
return item;
return item.entity;
}
}
plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
+7 -6
View File
@@ -15,21 +15,22 @@
*/
import { NotFoundError } from '@backstage/backend-common';
import { EntitiesCatalog, Entity } from './types';
import { EntitiesCatalog } from './types';
import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
export class StaticEntitiesCatalog implements EntitiesCatalog {
private _entities: Entity[];
private _entities: DescriptorEnvelope[];
constructor(entities: Entity[]) {
constructor(entities: DescriptorEnvelope[]) {
this._entities = entities;
}
async entities(): Promise<Entity[]> {
async entities(): Promise<DescriptorEnvelope[]> {
return this._entities.slice();
}
async entity(name: string): Promise<Entity> {
const item = this._entities.find(e => e.name === name);
async entity(name: string): Promise<DescriptorEnvelope> {
const item = this._entities.find(e => e.metadata?.name === name);
if (!item) {
throw new NotFoundError(`Found no entity with name ${name}`);
}
plugins/catalog-backend/src/catalog/types.ts
+3 -8
View File
@@ -15,20 +15,15 @@
*/
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
//
// Items
//
export type Entity = {
id: string;
locationId?: string;
name: string;
};
export type EntitiesCatalog = {
entities(): Promise<Entity[]>;
entity(id: string): Promise<Entity>;
entities(): Promise<DescriptorEnvelope[]>;
entity(id: string): Promise<DescriptorEnvelope>;
};
//
plugins/catalog-backend/src/database/Database.test.ts
+4 -4
View File
@@ -17,7 +17,7 @@
import knex from 'knex';
import path from 'path';
import { Database } from './Database';
import { AddDatabaseLocation, DatabaseLocation } from './types';
import { AddDatabaseLocation, DbLocationsRow } from './types';
describe('Database', () => {
const database = knex({
@@ -39,7 +39,7 @@ describe('Database', () => {
it('manages locations', async () => {
const db = new Database(database);
const input: AddDatabaseLocation = { type: 'a', target: 'b' };
const output: DatabaseLocation = {
const output: DbLocationsRow = {
id: expect.anything(),
type: 'a',
target: 'b',
@@ -64,10 +64,10 @@ describe('Database', () => {
// Prepare
const catalog = new Database(database);
const input: AddDatabaseLocation = { type: 'a', target: 'b' };
const output1: DatabaseLocation = await catalog.addLocation(input);
const output1: DbLocationsRow = await catalog.addLocation(input);
// Try to insert the same location
const output2: DatabaseLocation = await catalog.addLocation(input);
const output2: DbLocationsRow = await catalog.addLocation(input);
const locations = await catalog.locations();
// Output is the same
plugins/catalog-backend/src/database/Database.ts
+100 -27
View File
@@ -14,56 +14,131 @@
* limitations under the License.
*/
import { NotFoundError } from '@backstage/backend-common';
import { InputError, NotFoundError } from '@backstage/backend-common';
import Knex from 'knex';
import { v4 as uuidv4 } from 'uuid';
import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
import {
AddDatabaseEntity,
AddDatabaseLocation,
DatabaseEntity,
DatabaseLocation,
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
DbEntitiesRow,
DbEntityRequest,
DbEntityResponse,
DbLocationsRow,
} from './types';
function serializeMetadata(
metadata: DescriptorEnvelope['metadata'],
): DbEntitiesRow['metadata'] {
if (!metadata) {
return null;
}
const output = { ...metadata };
// TODO: delete output.uid;
// TODO: delete output.generation;
return JSON.stringify(output);
}
function serializeSpec(
spec: DescriptorEnvelope['spec'],
): DbEntitiesRow['spec'] {
if (!spec) {
return null;
}
return JSON.stringify(spec);
}
function entityRequestToDb(request: DbEntityRequest): DbEntitiesRow {
return {
id: '',
location_id: request.locationId || null,
api_version: request.entity.apiVersion,
kind: request.entity.kind,
name: request.entity.metadata?.name || null,
namespace: request.entity.metadata?.namespace || null,
metadata: serializeMetadata(request.entity.metadata),
spec: serializeSpec(request.entity.spec),
};
}
function entityDbToResponse(row: DbEntitiesRow): DbEntityResponse {
const entity: DescriptorEnvelope = {
apiVersion: row.api_version,
kind: row.kind,
metadata: {
// TODO: uid: row.id,
// TODO: generation: row.generation,
},
};
if (row.metadata) {
const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata'];
entity.metadata = { ...entity.metadata, ...metadata };
}
if (row.spec) {
const spec = JSON.parse(row.spec);
entity.spec = spec;
}
return {
locationId: row.location_id || undefined,
entity,
};
}
export class Database {
constructor(private readonly database: Knex) {}
async addOrUpdateEntity(entity: AddDatabaseEntity): Promise<void> {
async addOrUpdateEntity(request: DbEntityRequest): Promise<void> {
if (!request.entity.metadata?.name) {
throw new InputError(`Entities without names are not yet supported`);
}
const newRow = entityRequestToDb(request);
await this.database.transaction(async tx => {
// TODO(freben): Currently, several locations can compete for the same entity
// TODO(freben): If locationId is unset in the input, it won't be overwritten - should we instead replace with null?
const count = await tx<DatabaseEntity>('entities')
.where({ name: entity.name })
.update({ ...entity });
const count = await tx<DbEntitiesRow>('entities')
.where({ name: request.entity.metadata?.name })
.update({
...newRow,
id: undefined,
});
if (!count) {
await tx<DatabaseEntity>('entities').insert({
...entity,
await tx<DbEntitiesRow>('entities').insert({
...newRow,
id: uuidv4(),
});
}
});
}
async entities(): Promise<DatabaseEntity[]> {
return await this.database<DatabaseEntity>('entities')
async entities(): Promise<DbEntityResponse[]> {
const items = await this.database<DbEntitiesRow>('entities')
.orderBy('name')
.select();
return items.map(entityDbToResponse);
}
async entity(name: string): Promise<DatabaseEntity> {
const items = await this.database<DatabaseEntity>('entities')
async entity(name: string): Promise<DbEntityResponse> {
const items = await this.database<DbEntitiesRow>('entities')
.where({ name })
.select();
if (!items.length) {
throw new NotFoundError(`Found no entity with name ${name}`);
}
return items[0];
return entityDbToResponse(items[0]);
}
async addLocation(location: AddDatabaseLocation): Promise<DatabaseLocation> {
return await this.database.transaction<DatabaseLocation>(async tx => {
const existingLocation = await tx<DatabaseLocation>('locations')
async addLocation(location: AddDatabaseLocation): Promise<DbLocationsRow> {
return await this.database.transaction<DbLocationsRow>(async tx => {
const existingLocation = await tx<DbLocationsRow>('locations')
.where({
target: location.target,
})
@@ -75,20 +150,18 @@ export class Database {
const id = uuidv4();
const { type, target } = location;
await tx<DatabaseLocation>('locations').insert({
await tx<DbLocationsRow>('locations').insert({
id,
type,
target,
});
return (await tx<DatabaseLocation>('locations')
.where({ id })
.select())![0];
return (await tx<DbLocationsRow>('locations').where({ id }).select())![0];
});
}
async removeLocation(id: string): Promise<void> {
const result = await this.database<DatabaseLocation>('locations')
const result = await this.database<DbLocationsRow>('locations')
.where({ id })
.del();
@@ -97,8 +170,8 @@ export class Database {
}
}
async location(id: string): Promise<DatabaseLocation> {
const items = await this.database<DatabaseLocation>('locations')
async location(id: string): Promise<DbLocationsRow> {
const items = await this.database<DbLocationsRow>('locations')
.where({ id })
.select();
if (!items.length) {
@@ -107,8 +180,8 @@ export class Database {
return items[0];
}
async locations(): Promise<DatabaseLocation[]> {
return this.database<DatabaseLocation>('locations').select();
async locations(): Promise<DbLocationsRow[]> {
return this.database<DbLocationsRow>('locations').select();
}
async addLocationUpdateLogEvent(
plugins/catalog-backend/src/database/DatabaseManager.test.ts
+11 -9
View File
@@ -24,7 +24,7 @@ import {
} from '../ingestion';
import { Database } from './Database';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocation, DatabaseLocationUpdateLogStatus } from './types';
import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
describe('DatabaseManager', () => {
const logger = winston.createLogger({
@@ -60,7 +60,7 @@ describe('DatabaseManager', () => {
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
@@ -85,10 +85,12 @@ describe('DatabaseManager', () => {
expect(reader.read).toHaveBeenCalledTimes(1);
expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(db.addOrUpdateEntity).toHaveBeenCalledTimes(1);
expect(db.addOrUpdateEntity).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ locationId: '123', name: 'c1' }),
);
expect(db.addOrUpdateEntity).toHaveBeenNthCalledWith(1, {
locationId: '123',
entity: expect.objectContaining({
metadata: expect.objectContaining({ name: 'c1' }),
}),
});
});
it('logs successful updates', async () => {
@@ -100,7 +102,7 @@ describe('DatabaseManager', () => {
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
@@ -147,7 +149,7 @@ describe('DatabaseManager', () => {
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
@@ -197,7 +199,7 @@ describe('DatabaseManager', () => {
id: '123',
type: 'some',
target: 'thing',
} as DatabaseLocation,
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
plugins/catalog-backend/src/database/DatabaseManager.ts
+2 -5
View File
@@ -19,7 +19,7 @@ import path from 'path';
import { Logger } from 'winston';
import { DescriptorParser, LocationReader, ParserError } from '../ingestion';
import { Database } from './Database';
import { AddDatabaseEntity, DatabaseLocationUpdateLogStatus } from './types';
import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types';
export class DatabaseManager {
public static async createDatabase(database: Knex): Promise<Database> {
@@ -78,10 +78,7 @@ export class DatabaseManager {
}
try {
const entity = await parser.parse(readerItem.data);
const dbc: AddDatabaseEntity = {
locationId: location.id,
name: entity.metadata!.name!,
};
const dbc: DbEntityRequest = { locationId: location.id, entity };
await database.addOrUpdateEntity(dbc);
await DatabaseManager.logUpdateSuccess(
database,
plugins/catalog-backend/src/database/migrations/20200511113813_init.ts
+54 -27
View File
@@ -17,33 +17,60 @@
import * as Knex from 'knex';
export async function up(knex: Knex): Promise<any> {
return knex.schema
.createTable('locations', table => {
table.comment(
'Registered locations that shall be contiuously scanned for catalog item updates',
);
table.uuid('id').primary().comment('Auto-generated ID of the location');
table.string('type').notNullable().comment('The type of location');
table
.string('target')
.notNullable()
.comment('The actual target of the location');
})
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('locationId')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('name')
.unique()
.notNullable()
.comment('The external name of the entity, as used in references');
});
return (
knex.schema
//
// locations
//
.createTable('locations', table => {
table.comment(
'Registered locations that shall be contiuously scanned for catalog item updates',
);
table.uuid('id').primary().comment('Auto-generated ID of the location');
table.string('type').notNullable().comment('The type of location');
table
.string('target')
.notNullable()
.comment('The actual target of the location');
})
//
// entities
//
.createTable('entities', table => {
table.comment('All entities currently stored in the catalog');
table.uuid('id').primary().comment('Auto-generated ID of the entity');
table
.uuid('location_id')
.references('id')
.inTable('locations')
.nullable()
.comment('The location that originated the entity');
table
.string('api_version')
.notNullable()
.comment('The apiVersion field of the entity');
table
.string('kind')
.notNullable()
.comment('The kind field of the entity');
table
.string('name')
.nullable()
.comment('The metadata.name field of the entity');
table
.string('namespace')
.nullable()
.comment('The metadata.namespace field of the entity');
table
.string('metadata')
.nullable()
.comment('The entire metadata JSON blob of the entity');
table
.string('spec')
.nullable()
.comment('The entire spec JSON blob of the entity');
})
);
}
export async function down(knex: Knex): Promise<any> {
plugins/catalog-backend/src/database/types.ts
+16 -12
View File
@@ -15,26 +15,30 @@
*/
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
export type DatabaseEntity = {
export type DbEntitiesRow = {
id: string;
locationId?: string;
name: string;
location_id: string | null;
api_version: string;
kind: string;
name: string | null;
namespace: string | null;
metadata: string | null;
spec: string | null;
};
export type AddDatabaseEntity = {
export type DbEntityRequest = {
locationId?: string;
name: string;
entity: DescriptorEnvelope;
};
export const addDatabaseEntitySchema: yup.Schema<AddDatabaseEntity> = yup
.object({
locationId: yup.string().optional(),
name: yup.string().required(),
})
.noUnknown();
export type DbEntityResponse = {
locationId?: string;
entity: DescriptorEnvelope;
};
export type DatabaseLocation = {
export type DbLocationsRow = {
id: string;
type: string;
target: string;
plugins/catalog-backend/src/service/standaloneServer.ts
+12 -2
View File
@@ -31,8 +31,18 @@ export async function startStandaloneServer(
const logger = options.logger.child({ service: 'catalog-backend' });
const entitiesCatalog = new StaticEntitiesCatalog([
{ id: '1', name: 'entity1' },
{ id: '2', name: 'entity2' },
{
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
},
{
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c2' },
spec: { type: 'service' },
},
]);
logger.debug('Creating application...');