Merge pull request #1061 from spotify/freben/add-delete

Add routes for add/delete entity to the catalog

packages/backend/src/plugins/catalog.ts

@@ -41,7 +41,7 @@ export default async function ({ logger, database }: PluginEnvironment) {
     10000,
   );
 
-  const entitiesCatalog = new DatabaseEntitiesCatalog(db);
+  const entitiesCatalog = new DatabaseEntitiesCatalog(db, policy);
   const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion);
 
   return await createRouter({ entitiesCatalog, locationsCatalog, logger });

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Entity, EntityPolicy } from '@backstage/catalog-model';
+import type { Database } from '../database';
+import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
+
+describe('DatabaseEntitiesCatalog', () => {
+  let db: jest.Mocked<Database>;
+  let policy: EntityPolicy;
+
+  beforeEach(() => {
+    db = {
+      transaction: jest.fn(),
+      addEntity: jest.fn(),
+      updateEntity: jest.fn(),
+      entities: jest.fn(),
+      entity: jest.fn(),
+      removeEntity: jest.fn(),
+      addLocation: jest.fn(),
+      removeLocation: jest.fn(),
+      location: jest.fn(),
+      locations: jest.fn(),
+      locationHistory: jest.fn(),
+      addLocationUpdateLogEvent: jest.fn(),
+    };
+    db.transaction.mockImplementation(async f => f('tx'));
+    policy = { enforce: jest.fn(async x => x) };
+  });
+
+  describe('addOrUpdateEntity', () => {
+    it('adds when no given uid and no matching by name', async () => {
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+
+      db.entities.mockResolvedValue([]);
+      db.addEntity.mockResolvedValue({ entity });
+
+      const catalog = new DatabaseEntitiesCatalog(db, policy);
+      const result = await catalog.addOrUpdateEntity(entity);
+
+      expect(policy.enforce).toBeCalledWith(entity);
+      expect(db.entities).toHaveBeenCalledTimes(1);
+      expect(db.addEntity).toHaveBeenCalledTimes(1);
+      expect(result).toBe(entity);
+    });
+
+    it('updates when given uid', async () => {
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          uid: 'uuuu',
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+
+      db.entities.mockResolvedValue([]);
+      db.updateEntity.mockResolvedValue({ entity });
+
+      const catalog = new DatabaseEntitiesCatalog(db, policy);
+      const result = await catalog.addOrUpdateEntity(entity);
+
+      expect(policy.enforce).toBeCalledWith(entity);
+      expect(db.entities).toHaveBeenCalledTimes(0);
+      expect(db.updateEntity).toHaveBeenCalledTimes(1);
+      expect(result).toBe(entity);
+    });
+
+    it('update when no given uid and matching by name', async () => {
+      const added: Entity = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+      const existing: Entity = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+
+      db.entities.mockResolvedValue([{ entity: existing }]);
+      db.updateEntity.mockResolvedValue({ entity: added });
+
+      const catalog = new DatabaseEntitiesCatalog(db, policy);
+      const result = await catalog.addOrUpdateEntity(added);
+
+      expect(policy.enforce).toBeCalledWith(added);
+      expect(db.entities).toHaveBeenCalledTimes(1);
+      expect(db.updateEntity).toHaveBeenCalledTimes(1);
+      expect(result).toEqual(existing);
+    });
+  });
+});

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -14,12 +14,15 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
-import { Database } from '../database';
-import { EntitiesCatalog, EntityFilters } from './types';
+import type { Entity, EntityPolicy } from '@backstage/catalog-model';
+import type { Database, DbEntityResponse, EntityFilters } from '../database';
+import type { EntitiesCatalog } from './types';
 
 export class DatabaseEntitiesCatalog implements EntitiesCatalog {
-  constructor(private readonly database: Database) {}
+  constructor(
+    private readonly database: Database,
+    private readonly policy: EntityPolicy,
+  ) {}
 
   async entities(filters?: EntityFilters): Promise<Entity[]> {
     const items = await this.database.transaction(tx =>
@@ -41,19 +44,59 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
     name: string,
     namespace: string | undefined,
   ): Promise<Entity | undefined> {
-    const matches = await this.database.transaction(tx =>
-      this.database.entities(tx, [
-        { key: 'kind', values: [kind] },
-        { key: 'name', values: [name] },
-        {
-          key: 'namespace',
-          values:
-            !namespace || namespace === 'default'
-              ? [null, 'default']
-              : [namespace],
-        },
-      ]),
+    return await this.database.transaction(tx =>
+      this.entityByNameInternal(tx, kind, name, namespace),
     );
+  }
+
+  async addOrUpdateEntity(entity: Entity): Promise<Entity> {
+    await this.policy.enforce(entity);
+    return await this.database.transaction(async tx => {
+      let response: DbEntityResponse;
+
+      if (entity.metadata.uid) {
+        response = await this.database.updateEntity(tx, { entity });
+      } else {
+        const existing = await this.entityByNameInternal(
+          tx,
+          entity.kind,
+          entity.metadata.name,
+          entity.metadata.namespace,
+        );
+        if (existing) {
+          response = await this.database.updateEntity(tx, { entity });
+        } else {
+          response = await this.database.addEntity(tx, { entity });
+        }
+      }
+
+      return response.entity;
+    });
+  }
+
+  async removeEntityByUid(uid: string): Promise<void> {
+    return await this.database.transaction(async tx => {
+      await this.database.removeEntity(tx, uid);
+    });
+  }
+
+  private async entityByNameInternal(
+    tx: unknown,
+    kind: string,
+    name: string,
+    namespace: string | undefined,
+  ): Promise<Entity | undefined> {
+    const matches = await this.database.entities(tx, [
+      { key: 'kind', values: [kind] },
+      { key: 'name', values: [name] },
+      {
+        key: 'namespace',
+        values:
+          !namespace || namespace === 'default'
+            ? [null, 'default']
+            : [namespace],
+      },
+    ]);
 
     return matches.length ? matches[0].entity : undefined;
   }

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts

@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 import { getVoidLogger } from '@backstage/backend-common';
-import { Entity } from '@backstage/catalog-model';
-import knex from 'knex';
+import type { Entity } from '@backstage/catalog-model';
+import Knex from 'knex';
 import path from 'path';
-import { Database } from '../database';
-import { IngestionModel } from '../ingestion/types';
+import { CommonDatabase } from '../database';
+import type { Database } from '../database';
+import type { IngestionModel } from '../ingestion/types';
 import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
 
 class MockIngestionModel implements IngestionModel {
@@ -36,12 +37,12 @@ class MockIngestionModel implements IngestionModel {
 }
 
 describe('DatabaseLocationsCatalog', () => {
-  const database = knex({
+  const knex = Knex({
     client: 'sqlite3',
     connection: ':memory:',
     useNullAsDefault: true,
   });
-  database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
+  knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
     resource.run('PRAGMA foreign_keys = ON', () => {});
   });
   let db: Database;
@@ -49,11 +50,11 @@ describe('DatabaseLocationsCatalog', () => {
   let ingestionModel: IngestionModel;
 
   beforeEach(async () => {
-    await database.migrate.latest({
+    await knex.migrate.latest({
       directory: path.resolve(__dirname, '../database/migrations'),
       loadExtensions: ['.ts'],
     });
-    db = new Database(database, getVoidLogger());
+    db = new CommonDatabase(knex, getVoidLogger());
     ingestionModel = new MockIngestionModel();
     catalog = new DatabaseLocationsCatalog(db, ingestionModel);
   });

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts

@@ -14,13 +14,14 @@
  * limitations under the License.
  */
 
-import { Database, DatabaseLocationUpdateLogEvent } from '../database';
+import type { Database } from '../database';
+import { DatabaseLocationUpdateLogEvent } from '../database/types';
 import { IngestionModel } from '../ingestion/types';
 import {
   AddLocation,
   Location,
-  LocationsCatalog,
   LocationResponse,
+  LocationsCatalog,
 } from './types';
 
 export class DatabaseLocationsCatalog implements LocationsCatalog {

plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts

@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-import { NotFoundError } from '@backstage/backend-common';
-import { Entity } from '@backstage/catalog-model';
+import type { Entity } from '@backstage/catalog-model';
 import lodash from 'lodash';
-import { EntitiesCatalog } from './types';
+import type { EntitiesCatalog } from './types';
 
 export class StaticEntitiesCatalog implements EntitiesCatalog {
   private _entities: Entity[];
@@ -32,10 +31,7 @@ export class StaticEntitiesCatalog implements EntitiesCatalog {
 
   async entityByUid(uid: string): Promise<Entity | undefined> {
     const item = this._entities.find(e => uid === e.metadata.uid);
-    if (!item) {
-      throw new NotFoundError('Entity cannot be found');
-    }
-    return lodash.cloneDeep(item);
+    return item ? lodash.cloneDeep(item) : undefined;
   }
 
   async entityByName(
@@ -49,9 +45,14 @@ export class StaticEntitiesCatalog implements EntitiesCatalog {
         name === e.metadata.name &&
         namespace === e.metadata.namespace,
     );
-    if (!item) {
-      throw new NotFoundError('Entity cannot be found');
-    }
-    return lodash.cloneDeep(item);
+    return item ? lodash.cloneDeep(item) : undefined;
+  }
+
+  async addOrUpdateEntity(): Promise<Entity> {
+    throw new Error('Not supported');
+  }
+
+  async removeEntityByUid(): Promise<void> {
+    throw new Error('Not supported');
   }
 }

plugins/catalog-backend/src/catalog/index.ts

@@ -14,7 +14,13 @@
  * limitations under the License.
  */
 
-export * from './DatabaseEntitiesCatalog';
-export * from './DatabaseLocationsCatalog';
-export * from './StaticEntitiesCatalog';
-export * from './types';
+export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
+export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
+export { StaticEntitiesCatalog } from './StaticEntitiesCatalog';
+export { addLocationSchema } from './types';
+export type {
+  AddLocation,
+  EntitiesCatalog,
+  Location,
+  LocationsCatalog,
+} from './types';

plugins/catalog-backend/src/catalog/types.ts

@@ -16,17 +16,12 @@
 
 import { Entity } from '@backstage/catalog-model';
 import * as yup from 'yup';
+import type { EntityFilters } from '../database';
 
 //
 // Entities
 //
 
-export type EntityFilter = {
-  key: string;
-  values: (string | null)[];
-};
-export type EntityFilters = EntityFilter[];
-
 export type EntitiesCatalog = {
   entities(filters?: EntityFilters): Promise<Entity[]>;
   entityByUid(uid: string): Promise<Entity | undefined>;
@@ -35,6 +30,8 @@ export type EntitiesCatalog = {
     namespace: string | undefined,
     name: string,
   ): Promise<Entity | undefined>;
+  addOrUpdateEntity(entity: Entity): Promise<Entity>;
+  removeEntityByUid(uid: string): Promise<void>;
 };
 
 //

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -19,33 +19,33 @@ import {
   getVoidLogger,
   NotFoundError,
 } from '@backstage/backend-common';
+import type { Entity } from '@backstage/catalog-model';
 import Knex from 'knex';
 import path from 'path';
-import {
+import { CommonDatabase } from './CommonDatabase';
+import { DatabaseLocationUpdateLogStatus } from './types';
+import type {
+  AddDatabaseLocation,
   DbEntityRequest,
   DbEntityResponse,
-  Database,
-  AddDatabaseLocation,
   DbLocationsRow,
   DbLocationsRowWithStatus,
-  DatabaseLocationUpdateLogStatus,
-} from '.';
-import { Entity } from '@backstage/catalog-model';
+} from './types';
 
-describe('Database', () => {
-  let database: Knex;
+describe('CommonDatabase', () => {
+  let knex: Knex;
   let entityRequest: DbEntityRequest;
   let entityResponse: DbEntityResponse;
 
   beforeEach(async () => {
-    database = Knex({
+    knex = Knex({
       client: 'sqlite3',
       connection: ':memory:',
       useNullAsDefault: true,
     });
 
-    await database.raw('PRAGMA foreign_keys = ON');
-    await database.migrate.latest({
+    await knex.raw('PRAGMA foreign_keys = ON');
+    await knex.migrate.latest({
       directory: path.resolve(__dirname, 'migrations'),
       loadExtensions: ['.ts'],
     });
@@ -86,7 +86,7 @@ describe('Database', () => {
   });
 
   it('manages locations', async () => {
-    const db = new Database(database, getVoidLogger());
+    const db = new CommonDatabase(knex, getVoidLogger());
     const input: AddDatabaseLocation = { type: 'a', target: 'b' };
     const output: DbLocationsRowWithStatus = {
       id: expect.anything(),
@@ -114,7 +114,7 @@ describe('Database', () => {
 
   it('instead of adding second location with the same target, returns existing one', async () => {
     // Prepare
-    const catalog = new Database(database, getVoidLogger());
+    const catalog = new CommonDatabase(knex, getVoidLogger());
     const input: AddDatabaseLocation = { type: 'a', target: 'b' };
     const output1: DbLocationsRow = await catalog.addLocation(input);
 
@@ -130,7 +130,7 @@ describe('Database', () => {
 
   describe('addEntity', () => {
     it('happy path: adds entity to empty database', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -139,7 +139,7 @@ describe('Database', () => {
     });
 
     it('rejects adding the same-named entity twice', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       await catalog.transaction(tx => catalog.addEntity(tx, entityRequest));
       await expect(
         catalog.transaction(tx => catalog.addEntity(tx, entityRequest)),
@@ -147,7 +147,7 @@ describe('Database', () => {
     });
 
     it('accepts adding the same-named entity twice if on different namespaces', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       entityRequest.entity.metadata.namespace = 'namespace1';
       await catalog.transaction(tx => catalog.addEntity(tx, entityRequest));
       entityRequest.entity.metadata.namespace = 'namespace2';
@@ -159,7 +159,7 @@ describe('Database', () => {
 
   describe('locationHistory', () => {
     it('outputs the history correctly', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const location: AddDatabaseLocation = { type: 'a', target: 'b' };
       const { id: locationId } = await catalog.addLocation(location);
 
@@ -198,7 +198,7 @@ describe('Database', () => {
 
   describe('updateEntity', () => {
     it('can read and no-op-update an entity', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -220,7 +220,7 @@ describe('Database', () => {
     });
 
     it('can update name if uid matches', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -232,7 +232,7 @@ describe('Database', () => {
     });
 
     it('can update fields if kind, name, and namespace match', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -246,7 +246,7 @@ describe('Database', () => {
     });
 
     it('rejects if kind, name, but not namespace match', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -262,7 +262,7 @@ describe('Database', () => {
     });
 
     it('fails to update an entity if etag does not match', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -275,7 +275,7 @@ describe('Database', () => {
     });
 
     it('fails to update an entity if generation does not match', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const added = await catalog.transaction(tx =>
         catalog.addEntity(tx, entityRequest),
       );
@@ -290,7 +290,7 @@ describe('Database', () => {
 
   describe('entities', () => {
     it('can get all entities with empty filters list', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const e1: Entity = {
         apiVersion: 'a',
         kind: 'k1',
@@ -325,7 +325,7 @@ describe('Database', () => {
     });
 
     it('can get all specific entities for matching filters (naive case)', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const entities: Entity[] = [
         { apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } },
         {
@@ -364,7 +364,7 @@ describe('Database', () => {
     });
 
     it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => {
-      const catalog = new Database(database, getVoidLogger());
+      const catalog = new CommonDatabase(knex, getVoidLogger());
       const entities: Entity[] = [
         { apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } },
         {

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -19,15 +19,15 @@ import {
   InputError,
   NotFoundError,
 } from '@backstage/backend-common';
-import { Entity, EntityMeta } from '@backstage/catalog-model';
+import type { Entity, EntityMeta } from '@backstage/catalog-model';
 import Knex from 'knex';
 import lodash from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
-import { Logger } from 'winston';
-import { EntityFilters } from '../catalog';
+import type { Logger } from 'winston';
 import { buildEntitySearch } from './search';
-import {
+import type {
   AddDatabaseLocation,
+  Database,
   DatabaseLocationUpdateLogEvent,
   DatabaseLocationUpdateLogStatus,
   DbEntitiesRow,
@@ -36,6 +36,7 @@ import {
   DbEntityResponse,
   DbLocationsRow,
   DbLocationsRowWithStatus,
+  EntityFilters,
 } from './types';
 
 function getStrippedMetadata(metadata: EntityMeta): EntityMeta {
@@ -121,27 +122,13 @@ function generateEtag(): string {
   return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
 }
 
-/**
- * An abstraction on top of the underlying database, wrapping the basic CRUD
- * needs.
- */
-export class Database {
+export class CommonDatabase implements Database {
   constructor(
     private readonly database: Knex,
     private readonly logger: Logger,
   ) {}
 
-  /**
-   * Runs a transaction.
-   *
-   * The callback is expected to make calls back into this class. When it
-   * completes, the transaction is closed.
-   *
-   * @param fn The callback that implements the transaction
-   */
-  async transaction<T>(
-    fn: (tx: Knex.Transaction<any, any>) => Promise<T>,
-  ): Promise<T> {
+  async transaction<T>(fn: (tx: unknown) => Promise<T>): Promise<T> {
     try {
       return await this.database.transaction<T>(fn);
     } catch (e) {
@@ -158,17 +145,12 @@ export class Database {
     }
   }
 
-  /**
-   * Adds a new entity to the catalog.
-   *
-   * @param tx An ongoing transaction
-   * @param request The entity being added
-   * @returns The added entity, with uid, etag and generation set
-   */
   async addEntity(
-    tx: Knex.Transaction<any, any>,
+    txOpaque: unknown,
     request: DbEntityRequest,
   ): Promise<DbEntityResponse> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
     if (request.entity.metadata.uid !== undefined) {
       throw new InputError('May not specify uid for new entities');
     } else if (request.entity.metadata.etag !== undefined) {
@@ -198,25 +180,12 @@ export class Database {
     return { locationId: request.locationId, entity: newEntity };
   }
 
-  /**
-   * Updates an existing entity in the catalog.
-   *
-   * The given entity must contain enough information to identify an already
-   * stored entity in the catalog - either by uid, or by kind + namespace +
-   * name. If no matching entity is found, the operation fails.
-   *
-   * If etag or generation are given, they are taken into account. Attempts to
-   * update a matching entity, but where the etag and/or generation are not
-   * equal to the passed values, will fail.
-   *
-   * @param tx An ongoing transaction
-   * @param request The entity being updated
-   * @returns The updated entity
-   */
   async updateEntity(
-    tx: Knex.Transaction<any, any>,
+    txOpaque: unknown,
     request: DbEntityRequest,
   ): Promise<DbEntityResponse> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
     const { kind } = request.entity;
     const {
       uid,
@@ -310,9 +279,11 @@ export class Database {
   }
 
   async entities(
-    tx: Knex.Transaction<any, any>,
+    txOpaque: unknown,
     filters?: EntityFilters,
   ): Promise<DbEntityResponse[]> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
     let builder = tx<DbEntitiesRow>('entities');
     for (const [index, filter] of (filters ?? []).entries()) {
       builder = builder
@@ -337,11 +308,13 @@ export class Database {
   }
 
   async entity(
-    tx: Knex.Transaction<any, any>,
+    txOpaque: unknown,
     kind: string,
     name: string,
     namespace?: string,
   ): Promise<DbEntityResponse | undefined> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
     const rows = await tx<DbEntitiesRow>('entities')
       .where({ kind, name, namespace: namespace || null })
       .select();
@@ -353,6 +326,16 @@ export class Database {
     return toEntityResponse(rows[0]);
   }
 
+  async removeEntity(txOpaque: unknown, uid: string): Promise<void> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
+    const result = await tx<DbEntitiesRow>('entities').where({ id: uid }).del();
+
+    if (!result) {
+      throw new NotFoundError(`Found no entity with ID ${uid}`);
+    }
+  }
+
   async addLocation(location: AddDatabaseLocation): Promise<DbLocationsRow> {
     return await this.database.transaction<DbLocationsRow>(async tx => {
       const existingLocation = await tx<DbLocationsRow>('locations')

plugins/catalog-backend/src/database/DatabaseManager.test.ts

@@ -15,16 +15,16 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
+import type { Entity, EntityPolicy } from '@backstage/catalog-model';
 import Knex from 'knex';
-import { Database } from './Database';
+import type { IngestionModel } from '../ingestion/types';
 import { DatabaseManager } from './DatabaseManager';
-import {
-  DatabaseLocationUpdateLogStatus,
+import { DatabaseLocationUpdateLogStatus } from './types';
+import type {
+  Database,
   DbLocationsRow,
   DbLocationsRowWithStatus,
 } from './types';
-import { EntityPolicy, Entity } from '@backstage/catalog-model';
-import { IngestionModel } from '..';
 
 describe('DatabaseManager', () => {
   describe('refreshLocations', () => {

plugins/catalog-backend/src/database/DatabaseManager.ts

@@ -14,25 +14,26 @@
  * limitations under the License.
  */
 
-import { Entity, EntityPolicy } from '@backstage/catalog-model';
+import type { Entity, EntityPolicy } from '@backstage/catalog-model';
 import Knex from 'knex';
 import lodash from 'lodash';
 import path from 'path';
 import { Logger } from 'winston';
-import { IngestionModel } from '../ingestion/types';
-import { Database } from './Database';
-import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types';
+import type { IngestionModel } from '../ingestion/types';
+import { CommonDatabase } from './CommonDatabase';
+import { DatabaseLocationUpdateLogStatus } from './types';
+import type { Database, DbEntityRequest } from './types';
 
 export class DatabaseManager {
   public static async createDatabase(
-    database: Knex,
+    knex: Knex,
     logger: Logger,
   ): Promise<Database> {
-    await database.migrate.latest({
+    await knex.migrate.latest({
       directory: path.resolve(__dirname, 'migrations'),
       loadExtensions: ['.js'],
     });
-    return new Database(database, logger);
+    return new CommonDatabase(knex, logger);
   }
 
   private static async logUpdateSuccess(
@@ -158,22 +159,58 @@ export class DatabaseManager {
     });
   }
 
-  private static entitiesAreEqual(first: Entity, second: Entity) {
-    const firstClone = lodash.cloneDeep(first);
-    const secondClone = lodash.cloneDeep(second);
+  private static entitiesAreEqual(previous: Entity, next: Entity) {
+    if (
+      previous.apiVersion !== next.apiVersion ||
+      previous.kind !== next.kind ||
+      !lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined
+    ) {
+      return false;
+    }
+
+    // Since the next annotations get merged into the previous, extract only
+    // the overlapping keys and check if their values match.
+    if (next.metadata.annotations) {
+      if (!previous.metadata.annotations) {
+        return false;
+      }
+      if (
+        !lodash.isEqual(
+          next.metadata.annotations,
+          lodash.pick(
+            previous.metadata.annotations,
+            Object.keys(next.metadata.annotations),
+          ),
+        )
+      ) {
+        return false;
+      }
+    }
+
+    const e1 = lodash.cloneDeep(previous);
+    const e2 = lodash.cloneDeep(next);
+
+    if (!e1.metadata.labels) {
+      e1.metadata.labels = {};
+    }
+    if (!e2.metadata.labels) {
+      e2.metadata.labels = {};
+    }
 
     // Remove generated fields
-    if (firstClone.metadata) {
-      delete firstClone.metadata.uid;
-      delete firstClone.metadata.etag;
-      delete firstClone.metadata.generation;
-    }
-    if (secondClone.metadata) {
-      delete secondClone.metadata.uid;
-      delete secondClone.metadata.etag;
-      delete secondClone.metadata.generation;
-    }
+    delete e1.metadata.uid;
+    delete e1.metadata.etag;
+    delete e1.metadata.generation;
+    delete e2.metadata.uid;
+    delete e2.metadata.etag;
+    delete e2.metadata.generation;
 
-    return lodash.isEqual(firstClone, secondClone);
+    // Remove already compared things
+    delete e1.metadata.annotations;
+    delete e1.spec;
+    delete e2.metadata.annotations;
+    delete e2.spec;
+
+    return lodash.isEqual(e1, e2);
   }
 }

plugins/catalog-backend/src/database/index.ts

@@ -14,6 +14,12 @@
  * limitations under the License.
  */
 
-export * from './Database';
-export * from './DatabaseManager';
-export * from './types';
+export { CommonDatabase } from './CommonDatabase';
+export { DatabaseManager } from './DatabaseManager';
+export type {
+  Database,
+  DbEntityRequest,
+  DbEntityResponse,
+  EntityFilter,
+  EntityFilters,
+} from './types';

plugins/catalog-backend/src/database/search.test.ts

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import type { Entity } from '@backstage/catalog-model';
 import { buildEntitySearch, visitEntityPart } from './search';
-import { DbEntitiesSearchRow } from './types';
+import type { DbEntitiesSearchRow } from './types';
 
 describe('search', () => {
   describe('visitEntityPart', () => {

plugins/catalog-backend/src/database/search.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
-import { DbEntitiesSearchRow } from './types';
+import type { Entity } from '@backstage/catalog-model';
+import type { DbEntitiesSearchRow } from './types';
 
 // Search entries that start with these prefixes, also get a shorthand without
 // that prefix

plugins/catalog-backend/src/database/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import type { Entity } from '@backstage/catalog-model';
 import * as yup from 'yup';
 
 export type DbEntitiesRow = {
@@ -83,3 +83,82 @@ export type DatabaseLocationUpdateLogEvent = {
   created_at?: string;
   message?: string;
 };
+
+export type EntityFilter = {
+  key: string;
+  values: (string | null)[];
+};
+export type EntityFilters = EntityFilter[];
+
+/**
+ * An abstraction on top of the underlying database, wrapping the basic CRUD
+ * needs.
+ */
+export type Database = {
+  /**
+   * Runs a transaction.
+   *
+   * The callback is expected to make calls back into this class. When it
+   * completes, the transaction is closed.
+   *
+   * @param fn The callback that implements the transaction
+   */
+  transaction<T>(fn: (tx: unknown) => Promise<T>): Promise<T>;
+
+  /**
+   * Adds a new entity to the catalog.
+   *
+   * @param tx An ongoing transaction
+   * @param request The entity being added
+   * @returns The added entity, with uid, etag and generation set
+   */
+  addEntity(tx: unknown, request: DbEntityRequest): Promise<DbEntityResponse>;
+
+  /**
+   * Updates an existing entity in the catalog.
+   *
+   * The given entity must contain enough information to identify an already
+   * stored entity in the catalog - either by uid, or by kind + namespace +
+   * name. If no matching entity is found, the operation fails.
+   *
+   * If etag or generation are given, they are taken into account. Attempts to
+   * update a matching entity, but where the etag and/or generation are not
+   * equal to the passed values, will fail.
+   *
+   * @param tx An ongoing transaction
+   * @param request The entity being updated
+   * @returns The updated entity
+   */
+  updateEntity(
+    tx: unknown,
+    request: DbEntityRequest,
+  ): Promise<DbEntityResponse>;
+
+  entities(tx: unknown, filters?: EntityFilters): Promise<DbEntityResponse[]>;
+
+  entity(
+    tx: unknown,
+    kind: string,
+    name: string,
+    namespace?: string,
+  ): Promise<DbEntityResponse | undefined>;
+
+  removeEntity(tx: unknown, uid: string): Promise<void>;
+
+  addLocation(location: AddDatabaseLocation): Promise<DbLocationsRow>;
+
+  removeLocation(id: string): Promise<void>;
+
+  location(id: string): Promise<DbLocationsRowWithStatus>;
+
+  locations(): Promise<DbLocationsRowWithStatus[]>;
+
+  locationHistory(id: string): Promise<DatabaseLocationUpdateLogEvent[]>;
+
+  addLocationUpdateLogEvent(
+    locationId: string,
+    status: DatabaseLocationUpdateLogStatus,
+    entityName?: string,
+    message?: string,
+  ): Promise<void>;
+};

plugins/catalog-backend/src/ingestion/IngestionModels.ts

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model';
+import { EntityPolicies, EntityPolicy } from '@backstage/catalog-model';
+import { DescriptorParsers } from './descriptor';
 import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types';
 import { LocationReader, LocationReaders } from './source';
 import { IngestionModel } from './types';
-import { DescriptorParsers } from './descriptor';
 
 export class IngestionModels implements IngestionModel {
   private readonly reader: LocationReader;

plugins/catalog-backend/src/service/router.test.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-import { getVoidLogger } from '@backstage/backend-common';
-import { Entity } from '@backstage/catalog-model';
+import { getVoidLogger, NotFoundError } from '@backstage/backend-common';
+import type { Entity } from '@backstage/catalog-model';
 import express from 'express';
 import request from 'supertest';
 import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog';
@@ -25,6 +25,9 @@ class MockEntitiesCatalog implements EntitiesCatalog {
   entities = jest.fn();
   entityByUid = jest.fn();
   entityByName = jest.fn();
+  addEntity = jest.fn();
+  addOrUpdateEntity = jest.fn();
+  removeEntityByUid = jest.fn();
 }
 
 class MockLocationsCatalog implements LocationsCatalog {
@@ -36,7 +39,7 @@ class MockLocationsCatalog implements LocationsCatalog {
 }
 
 describe('createRouter', () => {
-  describe('entities', () => {
+  describe('GET /entities', () => {
     it('happy path: lists entities', async () => {
       const entities: Entity[] = [
         { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
@@ -77,7 +80,7 @@ describe('createRouter', () => {
     });
   });
 
-  describe('entityByUid', () => {
+  describe('GET /entities/by-uid/:uid', () => {
     it('can fetch entity by uid', async () => {
       const entity: Entity = {
         apiVersion: 'a',
@@ -118,7 +121,7 @@ describe('createRouter', () => {
     });
   });
 
-  describe('entityByName', () => {
+  describe('GET /entities/by-name/:kind/:namespace/:name', () => {
     it('can fetch entity by name', async () => {
       const entity: Entity = {
         apiVersion: 'a',
@@ -160,7 +163,91 @@ describe('createRouter', () => {
     });
   });
 
-  describe('locations', () => {
+  describe('POST /entities', () => {
+    it('requires a body', async () => {
+      const catalog = new MockEntitiesCatalog();
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app)
+        .post('/entities')
+        .set('Content-Type', 'application/json')
+        .send();
+
+      expect(response.status).toEqual(400);
+      expect(response.text).toMatch(/body/);
+      expect(catalog.addOrUpdateEntity).not.toHaveBeenCalled();
+    });
+
+    it('passes the body down', async () => {
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+
+      const catalog = new MockEntitiesCatalog();
+      catalog.addOrUpdateEntity.mockResolvedValue(entity);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app)
+        .post('/entities')
+        .send(entity)
+        .set('Content-Type', 'application/json');
+
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual(entity);
+      expect(catalog.addOrUpdateEntity).toHaveBeenCalledTimes(1);
+      expect(catalog.addOrUpdateEntity).toHaveBeenNthCalledWith(1, entity);
+    });
+  });
+
+  describe('DELETE /entities/by-uid/:uid', () => {
+    it('can remove', async () => {
+      const catalog = new MockEntitiesCatalog();
+      catalog.removeEntityByUid.mockResolvedValue(undefined);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).delete('/entities/by-uid/apa');
+
+      expect(response.status).toEqual(204);
+      expect(catalog.removeEntityByUid).toHaveBeenCalledTimes(1);
+    });
+
+    it('responds with a 404 for missing entities', async () => {
+      const catalog = new MockEntitiesCatalog();
+      catalog.removeEntityByUid.mockRejectedValue(new NotFoundError('nope'));
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).delete('/entities/by-uid/apa');
+
+      expect(response.status).toEqual(404);
+      expect(catalog.removeEntityByUid).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('GET /locations', () => {
     it('happy path: lists locations', async () => {
       const locations: Location[] = [{ id: 'a', type: 'b', target: 'c' }];
 
@@ -178,7 +265,9 @@ describe('createRouter', () => {
       expect(response.status).toEqual(200);
       expect(response.body).toEqual(locations);
     });
+  });
 
+  describe('POST /locations', () => {
     it('rejects malformed locations', async () => {
       const location = ({
         id: 'a',

plugins/catalog-backend/src/service/router.ts

@@ -15,16 +15,17 @@
  */
 
 import { errorHandler, InputError } from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
 import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import {
   addLocationSchema,
   EntitiesCatalog,
-  EntityFilters,
   LocationsCatalog,
 } from '../catalog';
-import { validateRequestBody } from './util';
+import { EntityFilters } from '../database';
+import { requireRequestBody, validateRequestBody } from './util';
 
 export interface RouterOptions {
   entitiesCatalog?: EntitiesCatalog;
@@ -47,6 +48,11 @@ export async function createRouter(
         const entities = await entitiesCatalog.entities(filters);
         res.status(200).send(entities);
       })
+      .post('/entities', async (req, res) => {
+        const body = await requireRequestBody(req);
+        const result = await entitiesCatalog.addOrUpdateEntity(body as Entity);
+        res.status(200).send(result);
+      })
       .get('/entities/by-uid/:uid', async (req, res) => {
         const { uid } = req.params;
         const entity = await entitiesCatalog.entityByUid(uid);
@@ -55,6 +61,11 @@ export async function createRouter(
         }
         res.status(200).send(entity);
       })
+      .delete('/entities/by-uid/:uid', async (req, res) => {
+        const { uid } = req.params;
+        await entitiesCatalog.removeEntityByUid(uid);
+        res.status(204).send();
+      })
       .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
         const { kind, namespace, name } = req.params;
         const entity = await entitiesCatalog.entityByName(

plugins/catalog-backend/src/service/util.ts

@@ -16,12 +16,10 @@
 
 import { InputError } from '@backstage/backend-common';
 import { Request } from 'express';
+import lodash from 'lodash';
 import yup from 'yup';
 
-export async function validateRequestBody<T>(
-  req: Request,
-  schema: yup.Schema<T>,
-): Promise<T> {
+export async function requireRequestBody(req: Request): Promise<unknown> {
   const contentType = req.header('content-type');
   if (!contentType) {
     throw new InputError('Content-Type missing');
@@ -32,13 +30,27 @@ export async function validateRequestBody<T>(
   const body = req.body;
   if (!body) {
     throw new InputError('Missing request body');
+  } else if (!lodash.isPlainObject(body)) {
+    throw new InputError('Expected body to be a JSON object');
+  } else if (Object.keys(body).length === 0) {
+    // Because of how express.json() translates the empty body to {}
+    throw new InputError('Empty request body');
   }
 
+  return body;
+}
+
+export async function validateRequestBody<T>(
+  req: Request,
+  schema: yup.Schema<T>,
+): Promise<T> {
+  const body = await requireRequestBody(req);
+
   try {
     await schema.validate(body, { strict: true });
   } catch (e) {
     throw new InputError(`Malformed request: ${e}`);
   }
 
-  return body as T;
+  return (body as unknown) as T;
 }