diff --git a/packages/backend/package.json b/packages/backend/package.json index e4b77c1e41..51fe04e51d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -45,6 +45,9 @@ "typescript": "^3.9.2" }, "nodemonConfig": { - "watch": "./dist" + "watch": [ + "./dist", + "node_modules/@backstage*" + ] } } diff --git a/packages/catalog-model/src/setupTests.ts b/packages/catalog-model/src/setupTests.ts new file mode 100644 index 0000000000..f3b69cc361 --- /dev/null +++ b/packages/catalog-model/src/setupTests.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index b13d70de86..b5841c8c2c 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { Database } from '../database'; +import { Database, DatabaseLocationUpdateLogEvent } from '../database'; import { IngestionModel } from '../ingestion/types'; -import { AddLocation, Location, LocationsCatalog } from './types'; +import { + AddLocation, + Location, + LocationsCatalog, + LocationResponse, +} from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( @@ -50,13 +55,36 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { await this.database.removeLocation(id); } - async locations(): Promise { + async locations(): Promise { const items = await this.database.locations(); - return items; + return items.map(({ message, status, timestamp, ...data }) => ({ + currentStatus: { + message, + status, + timestamp, + }, + data, + })); } - async location(id: string): Promise { - const item = await this.database.location(id); - return item; + async locationHistory(id: string): Promise { + return this.database.locationHistory(id); + } + + async location(id: string): Promise { + const { + message, + status, + timestamp, + ...data + } = await this.database.location(id); + return { + currentStatus: { + message, + status, + timestamp, + }, + data, + }; } } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 2f8967cd1b..5f55de251f 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -41,12 +41,31 @@ export type EntitiesCatalog = { // Locations // +export type LocationUpdateStatus = { + timestamp: string | null; + status: string | null; + message: string | null; +}; +export type LocationUpdateLogEvent = { + id: string; + status: 'fail' | 'success'; + location_id: string; + entity_name: string; + created_at?: string; + message?: string; +}; + export type Location = { id: string; type: string; target: string; }; +export type LocationResponse = { + data: Location; + currentStatus: LocationUpdateStatus; +}; + export type AddLocation = { type: string; target: string; @@ -62,6 +81,7 @@ export const addLocationSchema: yup.Schema = yup export type LocationsCatalog = { addLocation(location: AddLocation): Promise; removeLocation(id: string): Promise; - locations(): Promise; - location(id: string): Promise; + locations(): Promise; + location(id: string): Promise; + locationHistory(id: string): Promise; }; diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index a808e7e567..6f15440fff 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -27,6 +27,8 @@ import { Database, AddDatabaseLocation, DbLocationsRow, + DbLocationsRowWithStatus, + DatabaseLocationUpdateLogStatus, } from '.'; import { Entity } from '@backstage/catalog-model'; @@ -74,7 +76,9 @@ describe('Database', () => { name: 'c', namespace: 'd', labels: { e: 'f' }, - annotations: { g: 'h' }, + annotations: { + g: 'h', + }, }, spec: { i: 'j' }, }, @@ -84,10 +88,13 @@ describe('Database', () => { it('manages locations', async () => { const db = new Database(database, getVoidLogger()); const input: AddDatabaseLocation = { type: 'a', target: 'b' }; - const output: DbLocationsRow = { + const output: DbLocationsRowWithStatus = { id: expect.anything(), type: 'a', target: 'b', + message: null, + status: null, + timestamp: null, }; await db.addLocation(input); @@ -118,7 +125,7 @@ describe('Database', () => { // Output is the same expect(output2).toEqual(output1); // Locations contain only one record - expect(locations).toEqual([output1]); + expect(locations[0]).toMatchObject(output1); }); describe('addEntity', () => { @@ -150,6 +157,45 @@ describe('Database', () => { }); }); + describe('locationHistory', () => { + it('outputs the history correctly', async () => { + const catalog = new Database(database, getVoidLogger()); + const location: AddDatabaseLocation = { type: 'a', target: 'b' }; + const { id: locationId } = await catalog.addLocation(location); + + await catalog.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.SUCCESS, + ); + await catalog.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.FAIL, + undefined, + 'Something went wrong', + ); + + const result = await catalog.locationHistory(locationId); + expect(result).toEqual([ + { + created_at: expect.anything(), + entity_name: null, + id: expect.anything(), + location_id: locationId, + message: null, + status: DatabaseLocationUpdateLogStatus.SUCCESS, + }, + { + created_at: expect.anything(), + entity_name: null, + id: expect.anything(), + location_id: locationId, + message: 'Something went wrong', + status: DatabaseLocationUpdateLogStatus.FAIL, + }, + ]); + }); + }); + describe('updateEntity', () => { it('can read and no-op-update an entity', async () => { const catalog = new Database(database, getVoidLogger()); diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index 26deb3362e..ea823a3da7 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -35,6 +35,7 @@ import { DbEntityRequest, DbEntityResponse, DbLocationsRow, + DbLocationsRowWithStatus, } from './types'; function getStrippedMetadata(metadata: EntityMeta): EntityMeta { @@ -182,6 +183,12 @@ export class Database { uid: generateUid(), etag: generateEtag(), generation: 1, + annotations: { + ...(newEntity.metadata?.annotations ?? {}), + ...(request.locationId + ? { 'backstage.io/managed-by-location': request.locationId } + : {}), + }, }; const newRow = toEntityRow(request.locationId, newEntity); @@ -378,18 +385,52 @@ export class Database { } } - async location(id: string): Promise { - const items = await this.database('locations') - .where({ id }) - .select(); + async location(id: string): Promise { + const items = await this.database('locations') + .where('locations.id', id) + .leftOuterJoin( + 'location_update_log_latest', + 'locations.id', + 'location_update_log_latest.location_id', + ) + .select('locations.*', { + status: 'location_update_log_latest.status', + timestamp: 'location_update_log_latest.created_at', + message: 'location_update_log_latest.message', + }); + if (!items.length) { throw new NotFoundError(`Found no location with ID ${id}`); } return items[0]; } - async locations(): Promise { - return this.database('locations').select(); + async locations(): Promise { + const locations = await this.database('locations') + .leftOuterJoin( + 'location_update_log_latest', + 'locations.id', + 'location_update_log_latest.location_id', + ) + .select('locations.*', { + status: 'location_update_log_latest.status', + timestamp: 'location_update_log_latest.created_at', + message: 'location_update_log_latest.message', + }); + + return locations; + } + + async locationHistory(id: string): Promise { + const result = await this.database( + 'location_update_log', + ) + .where('location_id', id) + .orderBy('created_at', 'desc') + .limit(10) + .select(); + + return result; } async addLocationUpdateLogEvent( @@ -402,7 +443,7 @@ export class Database { 'location_update_log', ).insert({ id: uuidv4(), - status: status, + status, location_id: locationId, entity_name: entityName, message, diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index a65bbb7811..6e18eaf419 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -18,7 +18,11 @@ import { getVoidLogger } from '@backstage/backend-common'; import Knex from 'knex'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; -import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; +import { + DatabaseLocationUpdateLogStatus, + DbLocationsRow, + DbLocationsRowWithStatus, +} from './types'; import { EntityPolicy, Entity } from '@backstage/catalog-model'; import { IngestionModel } from '..'; @@ -43,10 +47,13 @@ describe('DatabaseManager', () => { }); it('can update a single location', async () => { - const location: DbLocationsRow = { + const location: DbLocationsRowWithStatus = { id: '123', type: 'some', target: 'thing', + message: '', + status: DatabaseLocationUpdateLogStatus.SUCCESS, + timestamp: new Date(314159265).toISOString(), }; const desc: Entity = { apiVersion: 'backstage.io/v1beta1', @@ -54,6 +61,7 @@ describe('DatabaseManager', () => { metadata: { name: 'c1' }, spec: { type: 'service' }, }; + const tx = (undefined as unknown) as Knex.Transaction; const db = ({ diff --git a/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts b/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts new file mode 100644 index 0000000000..6da9eeed9b --- /dev/null +++ b/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + // Need to first order by date of creation + const query = knex + .select() + .from('location_update_log') + .orderBy('location_update_log.created_at', 'desc'); + + // And only then to do the grouping to get the latest per location + const groupedQuery = knex(query).groupBy('location_id').select(); + + await knex.schema.raw( + `CREATE VIEW location_update_log_latest AS ${groupedQuery.toString()};`, + ); +} + +export async function down(knex: Knex): Promise { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`); +} diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 9b5a91d249..0b9485242f 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -52,6 +52,12 @@ export type DbLocationsRow = { target: string; }; +export type DbLocationsRowWithStatus = DbLocationsRow & { + status: string | null; + timestamp: string | null; + message: string | null; +}; + export type AddDatabaseLocation = { type: string; target: string; diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index f5675e6516..8c7946816c 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -32,6 +32,7 @@ class MockLocationsCatalog implements LocationsCatalog { removeLocation = jest.fn(); locations = jest.fn(); location = jest.fn(); + locationHistory = jest.fn(); } describe('createRouter', () => { diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index 0678cd3aef..32bba44984 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -22,7 +22,7 @@ import { addLocationSchema, EntitiesCatalog, EntityFilters, - LocationsCatalog + LocationsCatalog, } from '../catalog'; import { validateRequestBody } from './util'; @@ -84,6 +84,11 @@ export async function createRouter( const output = await locationsCatalog.locations(); res.status(200).send(output); }) + .get('/locations/:id/history', async (req, res) => { + const { id } = req.params; + const output = await locationsCatalog.locationHistory(id); + res.status(200).send(output); + }) .get('/locations/:id', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.location(id); diff --git a/plugins/catalog/src/api/CatalogClient.ts b/plugins/catalog/src/api/CatalogClient.ts new file mode 100644 index 0000000000..42ef1c4da5 --- /dev/null +++ b/plugins/catalog/src/api/CatalogClient.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogApi } from './types'; +import { DescriptorEnvelope } from '../types'; + +export class CatalogClient implements CatalogApi { + private apiOrigin: string; + private basePath: string; + constructor({ + apiOrigin, + basePath, + }: { + apiOrigin: string; + basePath: string; + }) { + this.apiOrigin = apiOrigin; + this.basePath = basePath; + } + async getEntities(): Promise { + const response = await fetch(`${this.apiOrigin}${this.basePath}/entities`); + return await response.json(); + } + async getEntityByName(name: string): Promise { + const response = await fetch( + `${this.apiOrigin}${this.basePath}/entities/by-name/Component/default/${name}`, + ); + const entity = await response.json(); + if (entity) return entity; + throw new Error(`'Entity not found: ${name}`); + } +} diff --git a/plugins/catalog/src/types.ts b/plugins/catalog/src/types.ts new file mode 100644 index 0000000000..0f91e1ed8e --- /dev/null +++ b/plugins/catalog/src/types.ts @@ -0,0 +1,159 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope { + spec: { + type: string; + }; +} + +export type ComponentDescriptor = ComponentDescriptorV1beta1; + +/** + * Metadata fields common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + */ +export type EntityMeta = { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace, for any given kind. + */ + name: string; + + /** + * The short description of the entity. + * + * A a human readable string. + */ + description: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: Record; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: Record; +}; + +/** + * The format envelope that's common to all versions/kinds. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type DescriptorEnvelope = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Optional metadata related to the entity. + */ + metadata: EntityMeta; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Parses and validates descriptors. + * + * The output must be validated and well formed. + */ +export type DescriptorParser = { + /** + * Parses and validates a single raw descriptor. + * + * @param descriptor A raw descriptor object + * @returns A structure describing the parsed and validated descriptor + * @throws An Error if the descriptor was malformed + */ + parse(descriptor: object): Promise; +}; + +/** + * Parses and validates a single envelope into its materialized kind. + * + * These parsers may assume that the envelope is already validated and well + * formed. + */ +export type KindParser = { + /** + * Try to parse an envelope into a materialized kind. + * + * @param envelope A valid descriptor envelope + * @returns A materialized type, or undefined if the given version/kind is + * not meant to be handled by this parser + * @throws An Error if the type was handled and found to not be properly + * formatted + */ + tryParse( + envelope: DescriptorEnvelope, + ): Promise; +};