diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 91cbd8c254..150b9d1826 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -23,6 +23,7 @@ import { LocationReaders, IngestionModels, runPeriodically, + HigherOrderOperations, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; import { EntityPolicies } from '@backstage/catalog-model'; @@ -32,7 +33,7 @@ export default async function createPlugin({ database, }: PluginEnvironment) { const policy = new EntityPolicies(); - const ingestion = new IngestionModels( + const ingestionModel = new IngestionModels( new LocationReaders(), new DescriptorParsers(), new EntityPolicies(), @@ -40,12 +41,22 @@ export default async function createPlugin({ const db = await DatabaseManager.createDatabase(database, logger); runPeriodically( - () => DatabaseManager.refreshLocations(db, ingestion, policy, logger), + () => DatabaseManager.refreshLocations(db, ingestionModel, policy, logger), 10000, ); - const entitiesCatalog = new DatabaseEntitiesCatalog(db, policy); - const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion); + const entitiesCatalog = new DatabaseEntitiesCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + const higherOrderOperation = new HigherOrderOperations( + entitiesCatalog, + locationsCatalog, + ingestionModel, + ); - return await createRouter({ entitiesCatalog, locationsCatalog, logger }); + return await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + logger, + }); } diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts index 1de5038112..b185075002 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import type { Entity, EntityPolicy } from '@backstage/catalog-model'; +import type { Entity } from '@backstage/catalog-model'; import type { Database } from '../database'; import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; describe('DatabaseEntitiesCatalog', () => { let db: jest.Mocked; - let policy: EntityPolicy; beforeEach(() => { db = { @@ -38,7 +37,6 @@ describe('DatabaseEntitiesCatalog', () => { addLocationUpdateLogEvent: jest.fn(), }; db.transaction.mockImplementation(async f => f('tx')); - policy = { enforce: jest.fn(async x => x) }; }); describe('addOrUpdateEntity', () => { @@ -55,10 +53,9 @@ describe('DatabaseEntitiesCatalog', () => { db.entities.mockResolvedValue([]); db.addEntity.mockResolvedValue({ entity }); - const catalog = new DatabaseEntitiesCatalog(db, policy); + const catalog = new DatabaseEntitiesCatalog(db); const result = await catalog.addOrUpdateEntity(entity); - expect(policy.enforce).toBeCalledWith(entity); expect(db.entities).toHaveBeenCalledTimes(1); expect(db.addEntity).toHaveBeenCalledTimes(1); expect(result).toBe(entity); @@ -78,10 +75,9 @@ describe('DatabaseEntitiesCatalog', () => { db.entities.mockResolvedValue([]); db.updateEntity.mockResolvedValue({ entity }); - const catalog = new DatabaseEntitiesCatalog(db, policy); + const catalog = new DatabaseEntitiesCatalog(db); const result = await catalog.addOrUpdateEntity(entity); - expect(policy.enforce).toBeCalledWith(entity); expect(db.entities).toHaveBeenCalledTimes(0); expect(db.updateEntity).toHaveBeenCalledTimes(1); expect(result).toBe(entity); @@ -108,10 +104,9 @@ describe('DatabaseEntitiesCatalog', () => { db.entities.mockResolvedValue([{ entity: existing }]); db.updateEntity.mockResolvedValue({ entity: added }); - const catalog = new DatabaseEntitiesCatalog(db, policy); + const catalog = new DatabaseEntitiesCatalog(db); const result = await catalog.addOrUpdateEntity(added); - expect(policy.enforce).toBeCalledWith(added); expect(db.entities).toHaveBeenCalledTimes(1); expect(db.updateEntity).toHaveBeenCalledTimes(1); expect(result).toEqual(existing); diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 606b6dc475..6a7a51a168 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,15 +14,12 @@ * limitations under the License. */ -import type { Entity, EntityPolicy } from '@backstage/catalog-model'; +import type { Entity } from '@backstage/catalog-model'; import type { Database, DbEntityResponse, EntityFilters } from '../database'; import type { EntitiesCatalog } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { - constructor( - private readonly database: Database, - private readonly policy: EntityPolicy, - ) {} + constructor(private readonly database: Database) {} async entities(filters?: EntityFilters): Promise { const items = await this.database.transaction(tx => @@ -53,7 +50,6 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { entity: Entity, locationId?: string, ): Promise { - await this.policy.enforce(entity); return await this.database.transaction(async tx => { let response: DbEntityResponse; diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 443b3bf6b0..907bdea2d5 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -13,73 +13,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; -import type { Entity } from '@backstage/catalog-model'; import Knex from 'knex'; import path from 'path'; import { CommonDatabase } from '../database'; -import type { Database } from '../database'; -import type { IngestionModel } from '../ingestion/types'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; -class MockIngestionModel implements IngestionModel { - readLocation = jest.fn(async (type: string, target: string) => { - if (type !== 'valid_type') { - throw new Error(`Unknown location type ${type}`); - } - if (target === 'valid_target') { - return [{ type: 'data', data: {} as Entity } as const]; - } - throw new Error( - `Can't read location at ${target} with error: Something is broken`, - ); - }); -} - describe('DatabaseLocationsCatalog', () => { - const knex = Knex({ - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }); - knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }); - let db: Database; let catalog: DatabaseLocationsCatalog; - let ingestionModel: IngestionModel; beforeEach(async () => { + const knex = Knex({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); await knex.migrate.latest({ directory: path.resolve(__dirname, '../database/migrations'), loadExtensions: ['.ts'], }); - db = new CommonDatabase(knex, getVoidLogger()); - ingestionModel = new MockIngestionModel(); - catalog = new DatabaseLocationsCatalog(db, ingestionModel); + const db = new CommonDatabase(knex, getVoidLogger()); + catalog = new DatabaseLocationsCatalog(db); }); - it('resolves to location with id', async () => { - return expect( - catalog.addLocation({ type: 'valid_type', target: 'valid_target' }), - ).resolves.toEqual({ - id: expect.anything(), + it('can add a location', async () => { + const location = { + id: 'dd12620d-0436-422f-93bd-929aa0788123', type: 'valid_type', target: 'valid_target', - }); - }); - it('rejects for invalid type', async () => { - const type = 'invalid_type'; - return expect( - catalog.addLocation({ type, target: 'valid_target' }), - ).rejects.toThrow(/Unknown location type/); - }); - it('rejects for unreadable target ', async () => { - const target = 'invalid_target'; - return expect( - catalog.addLocation({ type: 'valid_type', target }), - ).rejects.toThrow( - `Can't read location at ${target} with error: Something is broken`, - ); + }; + await expect(catalog.addLocation(location)).resolves.toEqual(location); + await expect( + catalog.location('dd12620d-0436-422f-93bd-929aa0788123'), + ).resolves.toEqual(expect.objectContaining({ data: location })); + await expect(catalog.locations()).resolves.toEqual([ + expect.objectContaining({ data: location }), + ]); }); }); diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index ae910e0b9b..2a5208c95a 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -16,38 +16,12 @@ import type { Database } from '../database'; import { DatabaseLocationUpdateLogEvent } from '../database/types'; -import { IngestionModel } from '../ingestion/types'; -import { - AddLocation, - Location, - LocationResponse, - LocationsCatalog, -} from './types'; +import { Location, LocationResponse, LocationsCatalog } from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { - constructor( - private readonly database: Database, - private readonly ingestionModel: IngestionModel, - ) {} - - async addLocation(location: AddLocation): Promise { - const outputs = await this.ingestionModel.readLocation( - location.type, - location.target, - ); - if (!outputs) { - throw new Error( - `Unknown location type ${location.type} ${location.target}`, - ); - } - outputs.forEach(output => { - if (output.type === 'error') { - throw new Error( - `Can't read location at ${location.target}, ${output.error}`, - ); - } - }); + constructor(private readonly database: Database) {} + async addLocation(location: Location): Promise { const added = await this.database.addLocation(location); return added; } diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index 6768268f34..dc0bb2e84a 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -17,10 +17,9 @@ export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; export { StaticEntitiesCatalog } from './StaticEntitiesCatalog'; -export { addLocationSchema } from './types'; export type { - AddLocation, EntitiesCatalog, Location, LocationsCatalog, + LocationSpec, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 2ef5fd9e56..a8c3c7a15e 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,7 +15,6 @@ */ import { Entity } from '@backstage/catalog-model'; -import * as yup from 'yup'; import type { EntityFilters } from '../database'; // @@ -52,31 +51,22 @@ export type LocationUpdateLogEvent = { message?: string; }; -export type Location = { - id: string; +export type LocationSpec = { type: string; target: string; }; +export type Location = { + id: string; +} & LocationSpec; + export type LocationResponse = { data: Location; currentStatus: LocationUpdateStatus; }; -export type AddLocation = { - type: string; - target: string; -}; - -export const addLocationSchema: yup.Schema = yup - .object({ - type: yup.string().required(), - target: yup.string().required(), - }) - .noUnknown(); - export type LocationsCatalog = { - addLocation(location: AddLocation): Promise; + addLocation(location: Location): Promise; removeLocation(id: string): Promise; locations(): Promise; location(id: string): Promise; diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index 3b783a7691..8f62969bd1 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -22,13 +22,12 @@ import { import type { Entity } from '@backstage/catalog-model'; import Knex from 'knex'; import path from 'path'; +import { Location } from '../catalog'; import { CommonDatabase } from './CommonDatabase'; import { DatabaseLocationUpdateLogStatus } from './types'; import type { - AddDatabaseLocation, DbEntityRequest, DbEntityResponse, - DbLocationsRow, DbLocationsRowWithStatus, } from './types'; @@ -87,9 +86,13 @@ describe('CommonDatabase', () => { it('manages locations', async () => { const db = new CommonDatabase(knex, getVoidLogger()); - const input: AddDatabaseLocation = { type: 'a', target: 'b' }; + const input: Location = { + id: 'dd12620d-0436-422f-93bd-929aa0788123', + type: 'a', + target: 'b', + }; const output: DbLocationsRowWithStatus = { - id: expect.anything(), + id: 'dd12620d-0436-422f-93bd-929aa0788123', type: 'a', target: 'b', message: null, @@ -112,22 +115,6 @@ describe('CommonDatabase', () => { ); }); - it('instead of adding second location with the same target, returns existing one', async () => { - // Prepare - const catalog = new CommonDatabase(knex, getVoidLogger()); - const input: AddDatabaseLocation = { type: 'a', target: 'b' }; - const output1: DbLocationsRow = await catalog.addLocation(input); - - // Try to insert the same location - const output2: DbLocationsRow = await catalog.addLocation(input); - const locations = await catalog.locations(); - - // Output is the same - expect(output2).toEqual(output1); - // Locations contain only one record - expect(locations[0]).toMatchObject(output1); - }); - describe('addEntity', () => { it('happy path: adds entity to empty database', async () => { const catalog = new CommonDatabase(knex, getVoidLogger()); @@ -160,27 +147,33 @@ describe('CommonDatabase', () => { describe('locationHistory', () => { it('outputs the history correctly', async () => { const catalog = new CommonDatabase(knex, getVoidLogger()); - const location: AddDatabaseLocation = { type: 'a', target: 'b' }; - const { id: locationId } = await catalog.addLocation(location); + const location: Location = { + id: 'dd12620d-0436-422f-93bd-929aa0788123', + type: 'a', + target: 'b', + }; + await catalog.addLocation(location); await catalog.addLocationUpdateLogEvent( - locationId, + 'dd12620d-0436-422f-93bd-929aa0788123', DatabaseLocationUpdateLogStatus.SUCCESS, ); await catalog.addLocationUpdateLogEvent( - locationId, + 'dd12620d-0436-422f-93bd-929aa0788123', DatabaseLocationUpdateLogStatus.FAIL, undefined, 'Something went wrong', ); - const result = await catalog.locationHistory(locationId); + const result = await catalog.locationHistory( + 'dd12620d-0436-422f-93bd-929aa0788123', + ); expect(result).toEqual([ { created_at: expect.anything(), entity_name: null, id: expect.anything(), - location_id: locationId, + location_id: 'dd12620d-0436-422f-93bd-929aa0788123', message: null, status: DatabaseLocationUpdateLogStatus.SUCCESS, }, @@ -188,7 +181,7 @@ describe('CommonDatabase', () => { created_at: expect.anything(), entity_name: null, id: expect.anything(), - location_id: locationId, + location_id: 'dd12620d-0436-422f-93bd-929aa0788123', message: 'Something went wrong', status: DatabaseLocationUpdateLogStatus.FAIL, }, diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index 2dac08b9bc..084f6ea662 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -26,7 +26,6 @@ import { v4 as uuidv4 } from 'uuid'; import type { Logger } from 'winston'; import { buildEntitySearch } from './search'; import type { - AddDatabaseLocation, Database, DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, @@ -38,6 +37,7 @@ import type { DbLocationsRowWithStatus, EntityFilters, } from './types'; +import { Location } from '../catalog'; function getStrippedMetadata(metadata: EntityMeta): EntityMeta { const output = lodash.cloneDeep(metadata); @@ -336,25 +336,15 @@ export class CommonDatabase implements Database { } } - async addLocation(location: AddDatabaseLocation): Promise { + async addLocation(location: Location): Promise { return await this.database.transaction(async tx => { - const existingLocation = await tx('locations') - .where({ target: location.target }) - .select(); - - if (existingLocation?.[0]) { - return existingLocation[0]; - } - - const id = uuidv4(); - const { type, target } = location; - await tx('locations').insert({ - id, - type, - target, - }); - - return (await tx('locations').where({ id }).select())![0]; + const row: DbLocationsRow = { + id: location.id, + type: location.type, + target: location.target, + }; + await tx('locations').insert(row); + return row; }); } diff --git a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts index 2ff06c4046..5f136670f4 100644 --- a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts +++ b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts @@ -26,7 +26,11 @@ export async function up(knex: Knex): Promise { table.comment( 'Registered locations that shall be contiuously scanned for catalog item updates', ); - table.uuid('id').primary().comment('Auto-generated ID of the location'); + table + .uuid('id') + .primary() + .notNullable() + .comment('Auto-generated ID of the location'); table.string('type').notNullable().comment('The type of location'); table .string('target') diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index f69a7353c7..4bcc66af1c 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -15,7 +15,7 @@ */ import type { Entity } from '@backstage/catalog-model'; -import * as yup from 'yup'; +import { Location } from '../catalog'; export type DbEntitiesRow = { id: string; @@ -58,18 +58,6 @@ export type DbLocationsRowWithStatus = DbLocationsRow & { message: string | null; }; -export type AddDatabaseLocation = { - type: string; - target: string; -}; - -export const addDatabaseLocationSchema: yup.Schema = yup - .object({ - type: yup.string().required(), - target: yup.string().required(), - }) - .noUnknown(); - export enum DatabaseLocationUpdateLogStatus { FAIL = 'fail', SUCCESS = 'success', @@ -145,7 +133,7 @@ export type Database = { removeEntity(tx: unknown, uid: string): Promise; - addLocation(location: AddDatabaseLocation): Promise; + addLocation(location: Location): Promise; removeLocation(id: string): Promise; diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts new file mode 100644 index 0000000000..0314556e27 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { v4 as uuidv4 } from 'uuid'; +import { + EntitiesCatalog, + Location, + LocationsCatalog, + LocationSpec, +} from '../catalog'; +import { IngestionModel } from '../ingestion'; +import { AddLocationResult, HigherOrderOperation } from './types'; + +const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; + +/** + * Placeholder for operations that span several catalogs and/or stretches out + * in time. + * + * TODO(freben): Find a better home for these, possibly refactoring to use the + * database more directly. + */ +export class HigherOrderOperations implements HigherOrderOperation { + private readonly entitiesCatalog: EntitiesCatalog; + private readonly locationsCatalog: LocationsCatalog; + private readonly ingestionModel: IngestionModel; + + constructor( + entitiesCatalog: EntitiesCatalog, + locationsCatalog: LocationsCatalog, + ingestionModel: IngestionModel, + ) { + this.entitiesCatalog = entitiesCatalog; + this.locationsCatalog = locationsCatalog; + this.ingestionModel = ingestionModel; + } + + /** + * Adds a single location to the catalog. + * + * The location is inspected and fetched, and all of the resulting data is + * validated. If everything goes well, the location and entities are stored + * in the catalog. + * + * If the location already existed, the old location is returned instead and + * the catalog is left unchanged. + * + * @param spec The location to add + */ + async addLocation(spec: LocationSpec): Promise { + // Attempt to find a previous location matching the spec + const previousLocations = await this.locationsCatalog.locations(); + const previousLocation = previousLocations.find( + l => spec.type === l.data.type && spec.target === l.data.target, + ); + const location: Location = previousLocation + ? previousLocation.data + : { + id: uuidv4(), + type: spec.type, + target: spec.target, + }; + + // Read the location fully, bailing on any errors + const readerOutput = await this.ingestionModel.readLocation( + location.type, + location.target, + ); + const inputEntities: Entity[] = []; + for (const entry of readerOutput) { + if (entry.type === 'error') { + throw new InputError( + `Failed to read location ${location.type} ${location.target}, ${entry.error}`, + ); + } else { + // Append the location reference annotation + entry.data.metadata.annotations = { + ...entry.data.metadata.annotations, + [LOCATION_ANNOTATION]: location.id, + }; + inputEntities.push(entry.data); + } + } + + // TODO(freben): At this point, we could detect orphaned entities, by way + // of having a LOCATION_ANNOTATION pointing to the location but not being + // in the entities list. But we aren't sure what to do about those yet. + + // Write + if (!previousLocation) { + await this.locationsCatalog.addLocation(location); + } + const outputEntities: Entity[] = []; + for (const entity of inputEntities) { + const out = await this.entitiesCatalog.addOrUpdateEntity( + entity, + location.id, + ); + outputEntities.push(out); + } + + return { location, entities: outputEntities }; + } +} diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index b6aceaecdd..93af856656 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -15,6 +15,7 @@ */ export * from './descriptor'; +export { HigherOrderOperations } from './HigherOrderOperations'; export { IngestionModels } from './IngestionModels'; export * from './source'; export type { IngestionModel } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 8878c2af5b..45dd0d9f74 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,8 +14,19 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; +import { Location, LocationSpec } from '../catalog'; import { ReaderOutput } from './descriptor/parsers/types'; +export type AddLocationResult = { + location: Location; + entities: Entity[]; +}; + export type IngestionModel = { readLocation(type: string, target: string): Promise; }; + +export type HigherOrderOperation = { + addLocation(spec: LocationSpec): Promise; +}; diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 4092ed1395..e03cb3dc98 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -18,42 +18,52 @@ import { getVoidLogger, NotFoundError } from '@backstage/backend-common'; import type { Entity } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; -import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; +import { EntitiesCatalog, LocationsCatalog, LocationSpec } from '../catalog'; +import { LocationResponse } from '../catalog/types'; +import { HigherOrderOperation } from '../ingestion/types'; import { createRouter } from './router'; -class MockEntitiesCatalog implements EntitiesCatalog { - entities = jest.fn(); - entityByUid = jest.fn(); - entityByName = jest.fn(); - addEntity = jest.fn(); - addOrUpdateEntity = jest.fn(); - removeEntityByUid = jest.fn(); -} - -class MockLocationsCatalog implements LocationsCatalog { - addLocation = jest.fn(); - removeLocation = jest.fn(); - locations = jest.fn(); - location = jest.fn(); - locationHistory = jest.fn(); -} - describe('createRouter', () => { + let entitiesCatalog: jest.Mocked; + let locationsCatalog: jest.Mocked; + let higherOrderOperation: jest.Mocked; + let app: express.Express; + + beforeEach(async () => { + entitiesCatalog = { + entities: jest.fn(), + entityByUid: jest.fn(), + entityByName: jest.fn(), + addOrUpdateEntity: jest.fn(), + removeEntityByUid: jest.fn(), + }; + locationsCatalog = { + addLocation: jest.fn(), + removeLocation: jest.fn(), + locations: jest.fn(), + location: jest.fn(), + locationHistory: jest.fn(), + }; + higherOrderOperation = { + addLocation: jest.fn(), + }; + const router = await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + logger: getVoidLogger(), + }); + app = express().use(router); + }); + describe('GET /entities', () => { it('happy path: lists entities', async () => { const entities: Entity[] = [ { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - const catalog = new MockEntitiesCatalog(); - catalog.entities.mockResolvedValueOnce(entities); + entitiesCatalog.entities.mockResolvedValueOnce(entities); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/entities'); expect(response.status).toEqual(200); @@ -61,18 +71,10 @@ describe('createRouter', () => { }); it('parses single and multiple request parameters and passes them down', async () => { - const catalog = new MockEntitiesCatalog(); - - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/entities?a=1&a=&a=3&b=4&c='); expect(response.status).toEqual(200); - expect(catalog.entities).toHaveBeenCalledWith([ + expect(entitiesCatalog.entities).toHaveBeenCalledWith([ { key: 'a', values: ['1', null, '3'] }, { key: 'b', values: ['4'] }, { key: 'c', values: [null] }, @@ -89,15 +91,8 @@ describe('createRouter', () => { name: 'c', }, }; - const catalog = new MockEntitiesCatalog(); - catalog.entityByUid.mockResolvedValue(entity); + entitiesCatalog.entityByUid.mockResolvedValue(entity); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/entities/by-uid/zzz'); expect(response.status).toEqual(200); @@ -105,15 +100,7 @@ describe('createRouter', () => { }); it('responds with a 404 for missing entities', async () => { - const catalog = new MockEntitiesCatalog(); - catalog.entityByUid.mockResolvedValue(undefined); - - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); + entitiesCatalog.entityByUid.mockResolvedValue(undefined); const response = await request(app).get('/entities/by-uid/zzz'); expect(response.status).toEqual(404); @@ -131,15 +118,8 @@ describe('createRouter', () => { namespace: 'd', }, }; - const catalog = new MockEntitiesCatalog(); - catalog.entityByName.mockResolvedValue(entity); + entitiesCatalog.entityByName.mockResolvedValue(entity); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/entities/by-name/b/d/c'); expect(response.status).toEqual(200); @@ -147,15 +127,8 @@ describe('createRouter', () => { }); it('responds with a 404 for missing entities', async () => { - const catalog = new MockEntitiesCatalog(); - catalog.entityByName.mockResolvedValue(undefined); + entitiesCatalog.entityByName.mockResolvedValue(undefined); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/entities/by-name//b/d/c'); expect(response.status).toEqual(404); @@ -165,13 +138,6 @@ describe('createRouter', () => { describe('POST /entities', () => { it('requires a body', async () => { - const catalog = new MockEntitiesCatalog(); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app) .post('/entities') .set('Content-Type', 'application/json') @@ -179,7 +145,7 @@ describe('createRouter', () => { expect(response.status).toEqual(400); expect(response.text).toMatch(/body/); - expect(catalog.addOrUpdateEntity).not.toHaveBeenCalled(); + expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled(); }); it('passes the body down', async () => { @@ -192,15 +158,8 @@ describe('createRouter', () => { }, }; - const catalog = new MockEntitiesCatalog(); - catalog.addOrUpdateEntity.mockResolvedValue(entity); + entitiesCatalog.addOrUpdateEntity.mockResolvedValue(entity); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app) .post('/entities') .send(entity) @@ -208,58 +167,46 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.body).toEqual(entity); - expect(catalog.addOrUpdateEntity).toHaveBeenCalledTimes(1); - expect(catalog.addOrUpdateEntity).toHaveBeenNthCalledWith(1, entity); + expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith( + 1, + entity, + ); }); }); describe('DELETE /entities/by-uid/:uid', () => { it('can remove', async () => { - const catalog = new MockEntitiesCatalog(); - catalog.removeEntityByUid.mockResolvedValue(undefined); + entitiesCatalog.removeEntityByUid.mockResolvedValue(undefined); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).delete('/entities/by-uid/apa'); expect(response.status).toEqual(204); - expect(catalog.removeEntityByUid).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1); }); it('responds with a 404 for missing entities', async () => { - const catalog = new MockEntitiesCatalog(); - catalog.removeEntityByUid.mockRejectedValue(new NotFoundError('nope')); + entitiesCatalog.removeEntityByUid.mockRejectedValue( + new NotFoundError('nope'), + ); - const router = await createRouter({ - entitiesCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).delete('/entities/by-uid/apa'); expect(response.status).toEqual(404); - expect(catalog.removeEntityByUid).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1); }); }); describe('GET /locations', () => { it('happy path: lists locations', async () => { - const locations: Location[] = [{ id: 'a', type: 'b', target: 'c' }]; + const locations: LocationResponse[] = [ + { + currentStatus: { timestamp: '', status: '', message: '' }, + data: { id: 'a', type: 'b', target: 'c' }, + }, + ]; + locationsCatalog.locations.mockResolvedValueOnce(locations); - const catalog = new MockLocationsCatalog(); - catalog.locations.mockResolvedValueOnce(locations); - - const router = await createRouter({ - locationsCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); const response = await request(app).get('/locations'); expect(response.status).toEqual(200); @@ -269,22 +216,33 @@ describe('createRouter', () => { describe('POST /locations', () => { it('rejects malformed locations', async () => { - const location = ({ - id: 'a', + const spec = ({ typez: 'b', target: 'c', - } as unknown) as Location; + } as unknown) as LocationSpec; - const catalog = new MockLocationsCatalog(); - const router = await createRouter({ - locationsCatalog: catalog, - logger: getVoidLogger(), - }); - - const app = express().use(router); - const response = await request(app).post('/locations').send(location); + const response = await request(app).post('/locations').send(spec); expect(response.status).toEqual(400); + expect(higherOrderOperation.addLocation).not.toHaveBeenCalled(); + }); + + it('passes the body down', async () => { + const spec: LocationSpec = { + type: 'b', + target: 'c', + }; + + higherOrderOperation.addLocation.mockResolvedValue({ + location: { id: 'a', ...spec }, + entities: [], + }); + + const response = await request(app).post('/locations').send(spec); + + expect(response.status).toEqual(201); + expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1); + expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec); }); }); }); diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index f5adbd84ca..95ea068f58 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -19,24 +19,30 @@ import { Entity } from '@backstage/catalog-model'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { - addLocationSchema, - EntitiesCatalog, - LocationsCatalog, -} from '../catalog'; +import * as yup from 'yup'; +import { EntitiesCatalog, LocationsCatalog, LocationSpec } from '../catalog'; import { EntityFilters } from '../database'; +import { HigherOrderOperation } from '../ingestion/types'; import { requireRequestBody, validateRequestBody } from './util'; export interface RouterOptions { entitiesCatalog?: EntitiesCatalog; locationsCatalog?: LocationsCatalog; + higherOrderOperation?: HigherOrderOperation; logger: Logger; } +const addLocationSchema = yup + .object({ + type: yup.string().required(), + target: yup.string().required(), + }) + .noUnknown(); + export async function createRouter( options: RouterOptions, ): Promise { - const { entitiesCatalog, locationsCatalog } = options; + const { entitiesCatalog, locationsCatalog, higherOrderOperation } = options; const router = Router(); router.use(express.json()); @@ -84,13 +90,16 @@ export async function createRouter( }); } + if (higherOrderOperation) { + router.post('/locations', async (req, res) => { + const input = await validateRequestBody(req, addLocationSchema); + const output = await higherOrderOperation.addLocation(input); + res.status(201).send(output); + }); + } + if (locationsCatalog) { router - .post('/locations', async (req, res) => { - const input = await validateRequestBody(req, addLocationSchema); - const output = await locationsCatalog.addLocation(input); - res.status(201).send(output); - }) .get('/locations', async (_req, res) => { const output = await locationsCatalog.locations(); res.status(200).send(output);