Merge pull request #1015 from spotify/mob/entity-list-and-item

Implement /entities with filters, /entities/by-id, /entities/by-name

packages/backend/package.json

@@ -40,8 +40,7 @@
     "@types/helmet": "^0.0.47",
     "jest": "^26.0.1",
     "tsc-watch": "^4.2.3",
-    "typescript": "^3.9.2",
-    "winston": "^3.2.1"
+    "typescript": "^3.9.2"
   },
   "nodemonConfig": {
     "watch": "./dist"

plugins/catalog-backend/package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@backstage/backend-common": "^0.1.1-alpha.6",
     "@types/node-fetch": "^2.5.7",
+    "@types/supertest": "^2.0.8",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
     "express": "^4.17.1",
@@ -40,6 +41,7 @@
     "@types/uuid": "^7.0.3",
     "@types/yup": "^0.28.2",
     "jest-fetch-mock": "^3.0.3",
+    "supertest": "^4.0.2",
     "tsc-watch": "^4.2.3"
   },
   "files": [

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -14,32 +14,47 @@
  * limitations under the License.
  */
 
-import { NotFoundError } from '@backstage/backend-common';
 import { Database } from '../database';
 import { DescriptorEnvelope } from '../ingestion/types';
-import { EntitiesCatalog } from './types';
+import { EntitiesCatalog, EntityFilters } from './types';
 
 export class DatabaseEntitiesCatalog implements EntitiesCatalog {
   constructor(private readonly database: Database) {}
 
-  async entities(): Promise<DescriptorEnvelope[]> {
+  async entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]> {
     const items = await this.database.transaction(tx =>
-      this.database.entities(tx),
+      this.database.entities(tx, filters),
     );
     return items.map(i => i.entity);
   }
 
-  async entity(
+  async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
+    const matches = await this.database.transaction(tx =>
+      this.database.entities(tx, [{ key: 'uid', values: [uid] }]),
+    );
+
+    return matches.length ? matches[0].entity : undefined;
+  }
+
+  async entityByName(
     kind: string,
     name: string,
     namespace: string | undefined,
   ): Promise<DescriptorEnvelope | undefined> {
-    const item = await this.database.transaction(tx =>
-      this.database.entity(tx, kind, name, namespace),
+    const matches = await this.database.transaction(tx =>
+      this.database.entities(tx, [
+        { key: 'kind', values: [kind] },
+        { key: 'name', values: [name] },
+        {
+          key: 'namespace',
+          values:
+            !namespace || namespace === 'default'
+              ? [null, 'default']
+              : [namespace],
+        },
+      ]),
     );
-    if (!item) {
-      throw new NotFoundError('Entity cannot be found');
-    }
-    return item.entity;
+
+    return matches.length ? matches[0].entity : undefined;
   }
 }

plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts

@@ -48,7 +48,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog {
   }
 
   async location(id: string): Promise<Location> {
-    const item = await this.location(id);
+    const item = await this.database.location(id);
     return item;
   }
 }

plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts

@@ -30,7 +30,15 @@ export class StaticEntitiesCatalog implements EntitiesCatalog {
     return lodash.cloneDeep(this._entities);
   }
 
-  async entity(
+  async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
+    const item = this._entities.find(e => uid === e.metadata?.uid);
+    if (!item) {
+      throw new NotFoundError('Entity cannot be found');
+    }
+    return lodash.cloneDeep(item);
+  }
+
+  async entityByName(
     kind: string,
     name: string,
     namespace: string | undefined,

plugins/catalog-backend/src/catalog/types.ts

@@ -18,15 +18,22 @@ import * as yup from 'yup';
 import { DescriptorEnvelope } from '../ingestion';
 
 //
-// Items
+// Entities
 //
 
+export type EntityFilter = {
+  key: string;
+  values: (string | null)[];
+};
+export type EntityFilters = EntityFilter[];
+
 export type EntitiesCatalog = {
-  entities(): Promise<DescriptorEnvelope[]>;
-  entity(
+  entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]>;
+  entityByUid(uid: string): Promise<DescriptorEnvelope | undefined>;
+  entityByName(
     kind: string,
-    name: string,
     namespace: string | undefined,
+    name: string,
   ): Promise<DescriptorEnvelope | undefined>;
 };
 

plugins/catalog-backend/src/database/Database.test.ts

@@ -21,6 +21,7 @@ import {
 } from '@backstage/backend-common';
 import Knex from 'knex';
 import path from 'path';
+import { DescriptorEnvelope } from '../ingestion';
 import { Database } from './Database';
 import {
   AddDatabaseLocation,
@@ -242,4 +243,112 @@ describe('Database', () => {
       ).rejects.toThrow(ConflictError);
     });
   });
+
+  describe('entities', () => {
+    it('can get all entities with empty filters list', async () => {
+      const catalog = new Database(database, getVoidLogger());
+      const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' };
+      const e2: DescriptorEnvelope = {
+        apiVersion: 'a',
+        kind: 'b',
+        spec: { c: null },
+      };
+      await catalog.transaction(async tx => {
+        await catalog.addEntity(tx, { entity: e1 });
+        await catalog.addEntity(tx, { entity: e2 });
+      });
+      const result = await catalog.transaction(async tx =>
+        catalog.entities(tx, []),
+      );
+      expect(result.length).toEqual(2);
+      expect(result).toEqual(
+        expect.arrayContaining([
+          { locationId: undefined, entity: expect.objectContaining(e1) },
+          { locationId: undefined, entity: expect.objectContaining(e2) },
+        ]),
+      );
+    });
+
+    it('can get all specific entities for matching filters (naive case)', async () => {
+      const catalog = new Database(database, getVoidLogger());
+      const entities: DescriptorEnvelope[] = [
+        { apiVersion: 'a', kind: 'b' },
+        {
+          apiVersion: 'a',
+          kind: 'b',
+          spec: { c: 'some' },
+        },
+        {
+          apiVersion: 'a',
+          kind: 'b',
+          spec: { c: null },
+        },
+      ];
+
+      await catalog.transaction(async tx => {
+        for (const entity of entities) {
+          await catalog.addEntity(tx, { entity });
+        }
+      });
+
+      await expect(
+        catalog.transaction(async tx =>
+          catalog.entities(tx, [
+            { key: 'kind', values: ['b'] },
+            { key: 'spec.c', values: ['some'] },
+          ]),
+        ),
+      ).resolves.toEqual([
+        { locationId: undefined, entity: expect.objectContaining(entities[1]) },
+      ]);
+    });
+
+    it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => {
+      const catalog = new Database(database, getVoidLogger());
+      const entities: DescriptorEnvelope[] = [
+        { apiVersion: 'a', kind: 'b' },
+        {
+          apiVersion: 'a',
+          kind: 'b',
+          spec: { c: 'some' },
+        },
+        {
+          apiVersion: 'a',
+          kind: 'b',
+          spec: { c: null },
+        },
+      ];
+
+      await catalog.transaction(async tx => {
+        for (const entity of entities) {
+          await catalog.addEntity(tx, { entity });
+        }
+      });
+
+      const rows = await catalog.transaction(async tx =>
+        catalog.entities(tx, [
+          { key: 'kind', values: ['b'] },
+          { key: 'spec.c', values: [null, 'some'] },
+        ]),
+      );
+
+      expect(rows.length).toEqual(3);
+      expect(rows).toEqual(
+        expect.arrayContaining([
+          {
+            locationId: undefined,
+            entity: expect.objectContaining(entities[0]),
+          },
+          {
+            locationId: undefined,
+            entity: expect.objectContaining(entities[1]),
+          },
+          {
+            locationId: undefined,
+            entity: expect.objectContaining(entities[2]),
+          },
+        ]),
+      );
+    });
+  });
 });

plugins/catalog-backend/src/database/Database.ts

@@ -23,6 +23,7 @@ import Knex from 'knex';
 import lodash from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
+import { EntityFilters } from '../catalog';
 import { DescriptorEnvelope, EntityMeta } from '../ingestion';
 import { buildEntitySearch } from './search';
 import {
@@ -186,11 +187,12 @@ export class Database {
     }
 
     const newEntity = lodash.cloneDeep(request.entity);
-    newEntity.metadata = Object.assign({}, newEntity.metadata, {
+    newEntity.metadata = {
+      ...newEntity.metadata,
       uid: generateUid(),
       etag: generateEtag(),
       generation: 1,
-    });
+    };
 
     const newRow = toEntityRow(request.locationId, newEntity);
     await tx<DbEntitiesRow>('entities').insert(newRow);
@@ -275,11 +277,12 @@ export class Database {
       ? oldRow.generation
       : oldRow.generation + 1;
     const newEntity = lodash.cloneDeep(request.entity);
-    newEntity.metadata = Object.assign({}, request.entity.metadata, {
+    newEntity.metadata = {
+      ...newEntity.metadata,
       uid: oldRow.id,
       etag: newEtag,
       generation: newGeneration,
-    });
+    };
 
     // Preserve annotations that were set on the old version of the entity,
     // unless the new version overwrites them
@@ -309,10 +312,30 @@ export class Database {
     return { locationId: request.locationId, entity: newEntity };
   }
 
-  async entities(tx: Knex.Transaction<any, any>): Promise<DbEntityResponse[]> {
-    const rows = await tx<DbEntitiesRow>('entities')
+  async entities(
+    tx: Knex.Transaction<any, any>,
+    filters?: EntityFilters,
+  ): Promise<DbEntityResponse[]> {
+    let builder = tx<DbEntitiesRow>('entities');
+    for (const [index, filter] of (filters ?? []).entries()) {
+      builder = builder
+        .leftOuterJoin(`entities_search as t${index}`, function join() {
+          this.on('entities.id', '=', `t${index}.entity_id`).onIn(
+            `t${index}.value`,
+            filter.values.filter(x => x),
+          );
+          if (filter.values.some(x => !x)) {
+            this.orOnNull(`t${index}.value`);
+          }
+        })
+        .where(`t${index}.key`, '=', filter.key);
+    }
+
+    const rows = await builder
       .orderBy('namespace', 'name')
-      .select();
+      .select('entities.*')
+      .groupBy('id');
+
     return rows.map(row => toEntityResponse(row));
   }
 
@@ -336,9 +359,7 @@ export class Database {
   async addLocation(location: AddDatabaseLocation): Promise<DbLocationsRow> {
     return await this.database.transaction<DbLocationsRow>(async tx => {
       const existingLocation = await tx<DbLocationsRow>('locations')
-        .where({
-          target: location.target,
-        })
+        .where({ target: location.target })
         .select();
 
       if (existingLocation?.[0]) {

plugins/catalog-backend/src/database/DatabaseManager.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
+import Knex from 'knex';
 import {
   ComponentDescriptor,
   DescriptorParser,
@@ -24,7 +25,6 @@ import {
 import { Database } from './Database';
 import { DatabaseManager } from './DatabaseManager';
 import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
-import Knex from 'knex';
 
 describe('DatabaseManager', () => {
   describe('refreshLocations', () => {

plugins/catalog-backend/src/service/router.test.ts

@@ -0,0 +1,198 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import express from 'express';
+import request from 'supertest';
+import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog';
+import { DescriptorEnvelope } from '../ingestion';
+import { createRouter } from './router';
+
+class MockEntitiesCatalog implements EntitiesCatalog {
+  entities = jest.fn();
+  entityByUid = jest.fn();
+  entityByName = jest.fn();
+}
+
+class MockLocationsCatalog implements LocationsCatalog {
+  addLocation = jest.fn();
+  removeLocation = jest.fn();
+  locations = jest.fn();
+  location = jest.fn();
+}
+
+describe('createRouter', () => {
+  describe('entities', () => {
+    it('happy path: lists entities', async () => {
+      const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }];
+
+      const catalog = new MockEntitiesCatalog();
+      catalog.entities.mockResolvedValueOnce(entities);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities');
+
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual(entities);
+    });
+
+    it('parses single and multiple request parameters and passes them down', async () => {
+      const catalog = new MockEntitiesCatalog();
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities?a=1&a=&a=3&b=4&c=');
+
+      expect(response.status).toEqual(200);
+      expect(catalog.entities).toHaveBeenCalledWith([
+        { key: 'a', values: ['1', null, '3'] },
+        { key: 'b', values: ['4'] },
+        { key: 'c', values: [null] },
+      ]);
+    });
+  });
+
+  describe('entityByUid', () => {
+    it('can fetch entity by uid', async () => {
+      const entity: DescriptorEnvelope = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+        },
+      };
+      const catalog = new MockEntitiesCatalog();
+      catalog.entityByUid.mockResolvedValue(entity);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities/by-uid/zzz');
+
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual(expect.objectContaining(entity));
+    });
+
+    it('responds with a 404 for missing entities', async () => {
+      const catalog = new MockEntitiesCatalog();
+      catalog.entityByUid.mockResolvedValue(undefined);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities/by-uid/zzz');
+
+      expect(response.status).toEqual(404);
+      expect(response.text).toMatch(/uid/);
+    });
+  });
+
+  describe('entityByName', () => {
+    it('can fetch entity by name', async () => {
+      const entity: DescriptorEnvelope = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'c',
+          namespace: 'd',
+        },
+      };
+      const catalog = new MockEntitiesCatalog();
+      catalog.entityByName.mockResolvedValue(entity);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities/by-name/b/d/c');
+
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual(expect.objectContaining(entity));
+    });
+
+    it('responds with a 404 for missing entities', async () => {
+      const catalog = new MockEntitiesCatalog();
+      catalog.entityByName.mockResolvedValue(undefined);
+
+      const router = await createRouter({
+        entitiesCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/entities/by-name//b/d/c');
+
+      expect(response.status).toEqual(404);
+      expect(response.text).toMatch(/name/);
+    });
+  });
+
+  describe('locations', () => {
+    it('happy path: lists locations', async () => {
+      const locations: Location[] = [{ id: 'a', type: 'b', target: 'c' }];
+
+      const catalog = new MockLocationsCatalog();
+      catalog.locations.mockResolvedValueOnce(locations);
+
+      const router = await createRouter({
+        locationsCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).get('/locations');
+
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual(locations);
+    });
+
+    it('rejects malformed locations', async () => {
+      const location = ({
+        id: 'a',
+        typez: 'b',
+        target: 'c',
+      } as unknown) as Location;
+
+      const catalog = new MockLocationsCatalog();
+      const router = await createRouter({
+        locationsCatalog: catalog,
+        logger: getVoidLogger(),
+      });
+
+      const app = express().use(router);
+      const response = await request(app).post('/locations').send(location);
+
+      expect(response.status).toEqual(400);
+    });
+  });
+});

plugins/catalog-backend/src/service/router.ts

@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+import { errorHandler, InputError } from '@backstage/backend-common';
 import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import {
   addLocationSchema,
   EntitiesCatalog,
+  EntityFilters,
   LocationsCatalog,
 } from '../catalog';
 import { validateRequestBody } from './util';
@@ -34,17 +36,43 @@ export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
   const { entitiesCatalog, locationsCatalog } = options;
+
   const router = Router();
+  router.use(express.json());
 
   if (entitiesCatalog) {
-    // Entities
-    router.get('/entities', async (_req, res) => {
-      const entities = await entitiesCatalog.entities();
-      res.status(200).send(entities);
-    });
+    router
+      .get('/entities', async (req, res) => {
+        const filters = translateQueryToEntityFilters(req);
+        const entities = await entitiesCatalog.entities(filters);
+        res.status(200).send(entities);
+      })
+      .get('/entities/by-uid/:uid', async (req, res) => {
+        const { uid } = req.params;
+        const entity = await entitiesCatalog.entityByUid(uid);
+        if (!entity) {
+          res.status(404).send(`No entity with uid ${uid}`);
+        }
+        res.status(200).send(entity);
+      })
+      .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
+        const { kind, namespace, name } = req.params;
+        const entity = await entitiesCatalog.entityByName(
+          kind,
+          name,
+          namespace,
+        );
+        if (!entity) {
+          res
+            .status(404)
+            .send(
+              `No entity with kind ${kind} namespace ${namespace} name ${name}`,
+            );
+        }
+        res.status(200).send(entity);
+      });
   }
 
-  // Locations
   if (locationsCatalog) {
     router
       .post('/locations', async (req, res) => {
@@ -68,5 +96,29 @@ export async function createRouter(
       });
   }
 
+  router.use(errorHandler());
   return router;
 }
+
+function translateQueryToEntityFilters(
+  request: express.Request,
+): EntityFilters {
+  const filters: EntityFilters = [];
+
+  for (const [key, valueOrValues] of Object.entries(request.query)) {
+    const values = Array.isArray(valueOrValues)
+      ? valueOrValues
+      : [valueOrValues];
+
+    if (values.some(v => typeof v !== 'string')) {
+      throw new InputError('Complex query parameters are not supported');
+    }
+
+    filters.push({
+      key,
+      values: values.map(v => v || null) as string[],
+    });
+  }
+
+  return filters;
+}