diff --git a/packages/backend/package.json b/packages/backend/package.json index 2c7b48f65f..9aedafecbb 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,8 +40,7 @@ "@types/helmet": "^0.0.47", "jest": "^26.0.1", "tsc-watch": "^4.2.3", - "typescript": "^3.9.2", - "winston": "^3.2.1" + "typescript": "^3.9.2" }, "nodemonConfig": { "watch": "./dist" diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9c0b066f4f..ce2a985efa 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -17,6 +17,7 @@ "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", "@types/node-fetch": "^2.5.7", + "@types/supertest": "^2.0.8", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -40,6 +41,7 @@ "@types/uuid": "^7.0.3", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", + "supertest": "^4.0.2", "tsc-watch": "^4.2.3" }, "files": [ diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 697da0ea35..9d0d8fcaf7 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,32 +14,47 @@ * limitations under the License. */ -import { NotFoundError } from '@backstage/backend-common'; import { Database } from '../database'; import { DescriptorEnvelope } from '../ingestion/types'; -import { EntitiesCatalog } from './types'; +import { EntitiesCatalog, EntityFilters } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { constructor(private readonly database: Database) {} - async entities(): Promise { + async entities(filters?: EntityFilters): Promise { const items = await this.database.transaction(tx => - this.database.entities(tx), + this.database.entities(tx, filters), ); return items.map(i => i.entity); } - async entity( + async entityByUid(uid: string): Promise { + const matches = await this.database.transaction(tx => + this.database.entities(tx, [{ key: 'uid', values: [uid] }]), + ); + + return matches.length ? matches[0].entity : undefined; + } + + async entityByName( kind: string, name: string, namespace: string | undefined, ): Promise { - const item = await this.database.transaction(tx => - this.database.entity(tx, kind, name, namespace), + const matches = await this.database.transaction(tx => + this.database.entities(tx, [ + { key: 'kind', values: [kind] }, + { key: 'name', values: [name] }, + { + key: 'namespace', + values: + !namespace || namespace === 'default' + ? [null, 'default'] + : [namespace], + }, + ]), ); - if (!item) { - throw new NotFoundError('Entity cannot be found'); - } - return item.entity; + + return matches.length ? matches[0].entity : undefined; } } diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 3f5739eed0..6e3c9ae306 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -48,7 +48,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { } async location(id: string): Promise { - const item = await this.location(id); + const item = await this.database.location(id); return item; } } diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 17f7603fb7..371cdf1f76 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -30,7 +30,15 @@ export class StaticEntitiesCatalog implements EntitiesCatalog { return lodash.cloneDeep(this._entities); } - async entity( + async entityByUid(uid: string): Promise { + const item = this._entities.find(e => uid === e.metadata?.uid); + if (!item) { + throw new NotFoundError('Entity cannot be found'); + } + return lodash.cloneDeep(item); + } + + async entityByName( kind: string, name: string, namespace: string | undefined, diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 92c059d25b..d5627ea86a 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -18,15 +18,22 @@ import * as yup from 'yup'; import { DescriptorEnvelope } from '../ingestion'; // -// Items +// Entities // +export type EntityFilter = { + key: string; + values: (string | null)[]; +}; +export type EntityFilters = EntityFilter[]; + export type EntitiesCatalog = { - entities(): Promise; - entity( + entities(filters?: EntityFilters): Promise; + entityByUid(uid: string): Promise; + entityByName( kind: string, - name: string, namespace: string | undefined, + name: string, ): Promise; }; diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index 0eb64916e5..a82385aae4 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -21,6 +21,7 @@ import { } from '@backstage/backend-common'; import Knex from 'knex'; import path from 'path'; +import { DescriptorEnvelope } from '../ingestion'; import { Database } from './Database'; import { AddDatabaseLocation, @@ -242,4 +243,112 @@ describe('Database', () => { ).rejects.toThrow(ConflictError); }); }); + + describe('entities', () => { + it('can get all entities with empty filters list', async () => { + const catalog = new Database(database, getVoidLogger()); + const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' }; + const e2: DescriptorEnvelope = { + apiVersion: 'a', + kind: 'b', + spec: { c: null }, + }; + await catalog.transaction(async tx => { + await catalog.addEntity(tx, { entity: e1 }); + await catalog.addEntity(tx, { entity: e2 }); + }); + const result = await catalog.transaction(async tx => + catalog.entities(tx, []), + ); + expect(result.length).toEqual(2); + expect(result).toEqual( + expect.arrayContaining([ + { locationId: undefined, entity: expect.objectContaining(e1) }, + { locationId: undefined, entity: expect.objectContaining(e2) }, + ]), + ); + }); + + it('can get all specific entities for matching filters (naive case)', async () => { + const catalog = new Database(database, getVoidLogger()); + const entities: DescriptorEnvelope[] = [ + { apiVersion: 'a', kind: 'b' }, + { + apiVersion: 'a', + kind: 'b', + spec: { c: 'some' }, + }, + { + apiVersion: 'a', + kind: 'b', + spec: { c: null }, + }, + ]; + + await catalog.transaction(async tx => { + for (const entity of entities) { + await catalog.addEntity(tx, { entity }); + } + }); + + await expect( + catalog.transaction(async tx => + catalog.entities(tx, [ + { key: 'kind', values: ['b'] }, + { key: 'spec.c', values: ['some'] }, + ]), + ), + ).resolves.toEqual([ + { locationId: undefined, entity: expect.objectContaining(entities[1]) }, + ]); + }); + + it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => { + const catalog = new Database(database, getVoidLogger()); + const entities: DescriptorEnvelope[] = [ + { apiVersion: 'a', kind: 'b' }, + { + apiVersion: 'a', + kind: 'b', + spec: { c: 'some' }, + }, + { + apiVersion: 'a', + kind: 'b', + spec: { c: null }, + }, + ]; + + await catalog.transaction(async tx => { + for (const entity of entities) { + await catalog.addEntity(tx, { entity }); + } + }); + + const rows = await catalog.transaction(async tx => + catalog.entities(tx, [ + { key: 'kind', values: ['b'] }, + { key: 'spec.c', values: [null, 'some'] }, + ]), + ); + + expect(rows.length).toEqual(3); + expect(rows).toEqual( + expect.arrayContaining([ + { + locationId: undefined, + entity: expect.objectContaining(entities[0]), + }, + { + locationId: undefined, + entity: expect.objectContaining(entities[1]), + }, + { + locationId: undefined, + entity: expect.objectContaining(entities[2]), + }, + ]), + ); + }); + }); }); diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index cd4ca731a3..614f7d09bf 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -23,6 +23,7 @@ import Knex from 'knex'; import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; +import { EntityFilters } from '../catalog'; import { DescriptorEnvelope, EntityMeta } from '../ingestion'; import { buildEntitySearch } from './search'; import { @@ -186,11 +187,12 @@ export class Database { } const newEntity = lodash.cloneDeep(request.entity); - newEntity.metadata = Object.assign({}, newEntity.metadata, { + newEntity.metadata = { + ...newEntity.metadata, uid: generateUid(), etag: generateEtag(), generation: 1, - }); + }; const newRow = toEntityRow(request.locationId, newEntity); await tx('entities').insert(newRow); @@ -275,11 +277,12 @@ export class Database { ? oldRow.generation : oldRow.generation + 1; const newEntity = lodash.cloneDeep(request.entity); - newEntity.metadata = Object.assign({}, request.entity.metadata, { + newEntity.metadata = { + ...newEntity.metadata, uid: oldRow.id, etag: newEtag, generation: newGeneration, - }); + }; // Preserve annotations that were set on the old version of the entity, // unless the new version overwrites them @@ -309,10 +312,30 @@ export class Database { return { locationId: request.locationId, entity: newEntity }; } - async entities(tx: Knex.Transaction): Promise { - const rows = await tx('entities') + async entities( + tx: Knex.Transaction, + filters?: EntityFilters, + ): Promise { + let builder = tx('entities'); + for (const [index, filter] of (filters ?? []).entries()) { + builder = builder + .leftOuterJoin(`entities_search as t${index}`, function join() { + this.on('entities.id', '=', `t${index}.entity_id`).onIn( + `t${index}.value`, + filter.values.filter(x => x), + ); + if (filter.values.some(x => !x)) { + this.orOnNull(`t${index}.value`); + } + }) + .where(`t${index}.key`, '=', filter.key); + } + + const rows = await builder .orderBy('namespace', 'name') - .select(); + .select('entities.*') + .groupBy('id'); + return rows.map(row => toEntityResponse(row)); } @@ -336,9 +359,7 @@ export class Database { async addLocation(location: AddDatabaseLocation): Promise { return await this.database.transaction(async tx => { const existingLocation = await tx('locations') - .where({ - target: location.target, - }) + .where({ target: location.target }) .select(); if (existingLocation?.[0]) { diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index 2da3c33a00..ca50f518cb 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import Knex from 'knex'; import { ComponentDescriptor, DescriptorParser, @@ -24,7 +25,6 @@ import { import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; -import Knex from 'knex'; describe('DatabaseManager', () => { describe('refreshLocations', () => { diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts new file mode 100644 index 0000000000..8c29362015 --- /dev/null +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import request from 'supertest'; +import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; +import { DescriptorEnvelope } from '../ingestion'; +import { createRouter } from './router'; + +class MockEntitiesCatalog implements EntitiesCatalog { + entities = jest.fn(); + entityByUid = jest.fn(); + entityByName = jest.fn(); +} + +class MockLocationsCatalog implements LocationsCatalog { + addLocation = jest.fn(); + removeLocation = jest.fn(); + locations = jest.fn(); + location = jest.fn(); +} + +describe('createRouter', () => { + describe('entities', () => { + it('happy path: lists entities', async () => { + const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }]; + + const catalog = new MockEntitiesCatalog(); + catalog.entities.mockResolvedValueOnce(entities); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(entities); + }); + + it('parses single and multiple request parameters and passes them down', async () => { + const catalog = new MockEntitiesCatalog(); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities?a=1&a=&a=3&b=4&c='); + + expect(response.status).toEqual(200); + expect(catalog.entities).toHaveBeenCalledWith([ + { key: 'a', values: ['1', null, '3'] }, + { key: 'b', values: ['4'] }, + { key: 'c', values: [null] }, + ]); + }); + }); + + describe('entityByUid', () => { + it('can fetch entity by uid', async () => { + const entity: DescriptorEnvelope = { + apiVersion: 'a', + kind: 'b', + metadata: { + name: 'c', + }, + }; + const catalog = new MockEntitiesCatalog(); + catalog.entityByUid.mockResolvedValue(entity); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities/by-uid/zzz'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(expect.objectContaining(entity)); + }); + + it('responds with a 404 for missing entities', async () => { + const catalog = new MockEntitiesCatalog(); + catalog.entityByUid.mockResolvedValue(undefined); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities/by-uid/zzz'); + + expect(response.status).toEqual(404); + expect(response.text).toMatch(/uid/); + }); + }); + + describe('entityByName', () => { + it('can fetch entity by name', async () => { + const entity: DescriptorEnvelope = { + apiVersion: 'a', + kind: 'b', + metadata: { + name: 'c', + namespace: 'd', + }, + }; + const catalog = new MockEntitiesCatalog(); + catalog.entityByName.mockResolvedValue(entity); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities/by-name/b/d/c'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(expect.objectContaining(entity)); + }); + + it('responds with a 404 for missing entities', async () => { + const catalog = new MockEntitiesCatalog(); + catalog.entityByName.mockResolvedValue(undefined); + + const router = await createRouter({ + entitiesCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/entities/by-name//b/d/c'); + + expect(response.status).toEqual(404); + expect(response.text).toMatch(/name/); + }); + }); + + describe('locations', () => { + it('happy path: lists locations', async () => { + const locations: Location[] = [{ id: 'a', type: 'b', target: 'c' }]; + + const catalog = new MockLocationsCatalog(); + catalog.locations.mockResolvedValueOnce(locations); + + const router = await createRouter({ + locationsCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).get('/locations'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(locations); + }); + + it('rejects malformed locations', async () => { + const location = ({ + id: 'a', + typez: 'b', + target: 'c', + } as unknown) as Location; + + const catalog = new MockLocationsCatalog(); + const router = await createRouter({ + locationsCatalog: catalog, + logger: getVoidLogger(), + }); + + const app = express().use(router); + const response = await request(app).post('/locations').send(location); + + expect(response.status).toEqual(400); + }); + }); +}); diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index af714e4026..c3318a7377 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -14,12 +14,14 @@ * limitations under the License. */ +import { errorHandler, InputError } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import { addLocationSchema, EntitiesCatalog, + EntityFilters, LocationsCatalog, } from '../catalog'; import { validateRequestBody } from './util'; @@ -34,17 +36,43 @@ export async function createRouter( options: RouterOptions, ): Promise { const { entitiesCatalog, locationsCatalog } = options; + const router = Router(); + router.use(express.json()); if (entitiesCatalog) { - // Entities - router.get('/entities', async (_req, res) => { - const entities = await entitiesCatalog.entities(); - res.status(200).send(entities); - }); + router + .get('/entities', async (req, res) => { + const filters = translateQueryToEntityFilters(req); + const entities = await entitiesCatalog.entities(filters); + res.status(200).send(entities); + }) + .get('/entities/by-uid/:uid', async (req, res) => { + const { uid } = req.params; + const entity = await entitiesCatalog.entityByUid(uid); + if (!entity) { + res.status(404).send(`No entity with uid ${uid}`); + } + res.status(200).send(entity); + }) + .get('/entities/by-name/:kind/:namespace/:name', async (req, res) => { + const { kind, namespace, name } = req.params; + const entity = await entitiesCatalog.entityByName( + kind, + name, + namespace, + ); + if (!entity) { + res + .status(404) + .send( + `No entity with kind ${kind} namespace ${namespace} name ${name}`, + ); + } + res.status(200).send(entity); + }); } - // Locations if (locationsCatalog) { router .post('/locations', async (req, res) => { @@ -68,5 +96,29 @@ export async function createRouter( }); } + router.use(errorHandler()); return router; } + +function translateQueryToEntityFilters( + request: express.Request, +): EntityFilters { + const filters: EntityFilters = []; + + for (const [key, valueOrValues] of Object.entries(request.query)) { + const values = Array.isArray(valueOrValues) + ? valueOrValues + : [valueOrValues]; + + if (values.some(v => typeof v !== 'string')) { + throw new InputError('Complex query parameters are not supported'); + } + + filters.push({ + key, + values: values.map(v => v || null) as string[], + }); + } + + return filters; +}