From 9ca8f8355f199992c8d37d80f3f095d7ce9e8abe Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Tue, 26 May 2020 16:40:41 +0200 Subject: [PATCH 1/5] feat: location update results --- .../src/catalog/DatabaseLocationsCatalog.ts | 41 ++++++++++-- plugins/catalog-backend/src/catalog/types.ts | 17 ++++- .../src/database/Database.test.ts | 53 ++++++++++++++- .../catalog-backend/src/database/Database.ts | 66 +++++++++++++++++-- .../src/database/DatabaseManager.test.ts | 12 +++- plugins/catalog-backend/src/database/types.ts | 6 ++ .../catalog-backend/src/ingestion/types.ts | 2 +- plugins/catalog-backend/src/service/router.ts | 5 ++ 8 files changed, 181 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 3f5739eed0..f571a20cfa 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -14,9 +14,15 @@ * limitations under the License. */ -import { Database } from '../database'; import { AddLocation, Location, LocationsCatalog } from './types'; import { LocationReader } from '../ingestion'; +import { Database, DatabaseLocationUpdateLogEvent } from '../database'; +import { + AddLocation, + LocationEnvelope, + Location, + LocationsCatalog, +} from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( @@ -42,13 +48,36 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { await this.database.removeLocation(id); } - async locations(): Promise { + async locations(): Promise { const items = await this.database.locations(); - return items; + return items.map(({ message, status, timestamp, ...data }) => ({ + lastUpdate: { + message, + status, + timestamp, + }, + data, + })); } - async location(id: string): Promise { - const item = await this.location(id); - return item; + async locationHistory(id: string): Promise { + return this.database.locationHistory(id); + } + + async location(id: string): Promise { + const { + message, + status, + timestamp, + ...data + } = await this.database.location(id); + return { + lastUpdate: { + message, + status, + timestamp, + }, + data, + }; } } diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 92c059d25b..ad7969405b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -16,6 +16,7 @@ import * as yup from 'yup'; import { DescriptorEnvelope } from '../ingestion'; +import { DatabaseLocationUpdateLogEvent } from '../database'; // // Items @@ -34,12 +35,23 @@ export type EntitiesCatalog = { // Locations // +export type status = { + timestamp: DatabaseLocationUpdateLogEvent['created_at'] | null; + status: DatabaseLocationUpdateLogEvent['status'] | null; + message: DatabaseLocationUpdateLogEvent['message'] | null; +}; + export type Location = { id: string; type: string; target: string; }; +export type LocationEnvelope = { + data: Location; + lastUpdate: status; +}; + export type AddLocation = { type: string; target: string; @@ -55,6 +67,7 @@ export const addLocationSchema: yup.Schema = yup export type LocationsCatalog = { addLocation(location: AddLocation): Promise; removeLocation(id: string): Promise; - locations(): Promise; - location(id: string): Promise; + locations(): Promise; + location(id: string): Promise; + locationHistory(id: string): Promise; }; diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index 0eb64916e5..b02bbab668 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -27,6 +27,8 @@ import { DbEntityRequest, DbEntityResponse, DbLocationsRow, + DbLocationsRowWithStatus, + DatabaseLocationUpdateLogStatus, } from './types'; describe('Database', () => { @@ -73,7 +75,10 @@ describe('Database', () => { name: 'c', namespace: 'd', labels: { e: 'f' }, - annotations: { g: 'h' }, + annotations: { + g: 'h', + 'backstage.io/managed-by-location': undefined, + }, }, spec: { i: 'j' }, }, @@ -83,10 +88,13 @@ describe('Database', () => { it('manages locations', async () => { const db = new Database(database, getVoidLogger()); const input: AddDatabaseLocation = { type: 'a', target: 'b' }; - const output: DbLocationsRow = { + const output: DbLocationsRowWithStatus = { id: expect.anything(), type: 'a', target: 'b', + message: null, + status: null, + timestamp: null, }; await db.addLocation(input); @@ -117,7 +125,7 @@ describe('Database', () => { // Output is the same expect(output2).toEqual(output1); // Locations contain only one record - expect(locations).toEqual([output1]); + expect(locations[0]).toMatchObject(output1); }); describe('addEntity', () => { @@ -149,6 +157,45 @@ describe('Database', () => { }); }); + describe('locationHistory', () => { + it('outputs the history correctly', async () => { + const catalog = new Database(database, getVoidLogger()); + const location: AddDatabaseLocation = { type: 'a', target: 'b' }; + const { id: locationId } = await catalog.addLocation(location); + + await catalog.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.SUCCESS, + ); + await catalog.addLocationUpdateLogEvent( + locationId, + DatabaseLocationUpdateLogStatus.FAIL, + undefined, + 'Something went wrong', + ); + + const result = await catalog.locationHistory(locationId); + expect(result).toEqual([ + { + created_at: expect.anything(), + entity_name: null, + id: expect.anything(), + location_id: locationId, + message: null, + status: DatabaseLocationUpdateLogStatus.SUCCESS, + }, + { + created_at: expect.anything(), + entity_name: null, + id: expect.anything(), + location_id: locationId, + message: 'Something went wrong', + status: DatabaseLocationUpdateLogStatus.FAIL, + }, + ]); + }); + }); + describe('updateEntity', () => { it('can read and no-op-update an entity', async () => { const catalog = new Database(database, getVoidLogger()); diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index cd4ca731a3..9e2b69bce2 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -34,6 +34,7 @@ import { DbEntityRequest, DbEntityResponse, DbLocationsRow, + DbLocationsRowWithStatus, } from './types'; function getStrippedMetadata(metadata: EntityMeta): EntityMeta { @@ -190,6 +191,10 @@ export class Database { uid: generateUid(), etag: generateEtag(), generation: 1, + annotations: { + ...(newEntity.metadata?.annotations ?? {}), + 'backstage.io/managed-by-location': request.locationId, + }, }); const newRow = toEntityRow(request.locationId, newEntity); @@ -367,18 +372,65 @@ export class Database { } } - async location(id: string): Promise { - const items = await this.database('locations') - .where({ id }) - .select(); + async location(id: string): Promise { + const items = await this.database('locations') + .where('locations.id', id) + .leftJoin( + 'location_update_log', + 'locations.id', + 'location_update_log.location_id', + ) + .orderBy('location_update_log.created_at', 'desc') + .select({ + status: 'location_update_log.status', + timestamp: 'location_update_log.created_at', + message: 'location_update_log.message', + id: 'locations.id', + type: 'locations.type', + target: 'locations.target', + }); + if (!items.length) { throw new NotFoundError(`Found no location with ID ${id}`); } return items[0]; } - async locations(): Promise { - return this.database('locations').select(); + async locations(): Promise { + const query = this.database + .select({ + status: 'location_update_log.status', + timestamp: 'location_update_log.created_at', + message: 'location_update_log.message', + id: 'locations.id', + type: 'locations.type', + target: 'locations.target', + }) + .from('locations') + .leftJoin( + 'location_update_log', + 'locations.id', + 'location_update_log.location_id', + ) + .orderBy('location_update_log.created_at', 'desc'); + + const result = await this.database(query) + .select() + .groupBy('id'); + + return result; + } + + async locationHistory(id: string): Promise { + const result = await this.database( + 'location_update_log', + ) + .where('location_id', id) + .orderBy('created_at', 'desc') + .limit(10) + .select(); + + return result; } async addLocationUpdateLogEvent( @@ -391,7 +443,7 @@ export class Database { 'location_update_log', ).insert({ id: uuidv4(), - status: status, + status, location_id: locationId, entity_name: entityName, message, diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index 2da3c33a00..efae9e3516 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -23,7 +23,11 @@ import { } from '../ingestion'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; -import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; +import { + DatabaseLocationUpdateLogStatus, + DbLocationsRow, + DbLocationsRowWithStatus, +} from './types'; import Knex from 'knex'; describe('DatabaseManager', () => { @@ -47,10 +51,13 @@ describe('DatabaseManager', () => { }); it('can update a single location', async () => { - const location: DbLocationsRow = { + const location: DbLocationsRowWithStatus = { id: '123', type: 'some', target: 'thing', + message: '', + status: DatabaseLocationUpdateLogStatus.SUCCESS, + timestamp: new Date(314159265).toISOString(), }; const desc: ComponentDescriptor = { apiVersion: 'backstage.io/v1beta1', @@ -58,6 +65,7 @@ describe('DatabaseManager', () => { metadata: { name: 'c1' }, spec: { type: 'service' }, }; + const tx = (undefined as unknown) as Knex.Transaction; const db = ({ diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 06e234545f..e20ba59517 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -52,6 +52,12 @@ export type DbLocationsRow = { target: string; }; +export type DbLocationsRowWithStatus = DbLocationsRow & { + status: DatabaseLocationUpdateLogEvent['status'] | null; + timestamp: DatabaseLocationUpdateLogEvent['created_at'] | null; + message: DatabaseLocationUpdateLogEvent['message'] | null; +}; + export type AddDatabaseLocation = { type: string; target: string; diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index e5083a65f1..87a6f19f1f 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -79,7 +79,7 @@ export type EntityMeta = { * Key/value pairs of non-identifying auxiliary information attached to the * entity. */ - annotations?: Record; + annotations?: Record; }; /** diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index af714e4026..80ba84d0f0 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -56,6 +56,11 @@ export async function createRouter( const output = await locationsCatalog.locations(); res.status(200).send(output); }) + .get('/locations/:id/history', async (req, res) => { + const { id } = req.params; + const output = await locationsCatalog.locationHistory(id); + res.status(200).send(output); + }) .get('/locations/:id', async (req, res) => { const { id } = req.params; const output = await locationsCatalog.location(id); From 0cae5dea3a4e4aa499187c4469c254d1650e980b Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Tue, 26 May 2020 16:44:49 +0200 Subject: [PATCH 2/5] fix: conflict resolution leftover --- plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index f571a20cfa..d79769a5a7 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { AddLocation, Location, LocationsCatalog } from './types'; import { LocationReader } from '../ingestion'; import { Database, DatabaseLocationUpdateLogEvent } from '../database'; import { From 9310ce5aee803ef95053d3dc912223e2c12c69f3 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 27 May 2020 10:04:55 +0200 Subject: [PATCH 3/5] fix: test --- plugins/catalog-backend/src/service/router.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 8c29362015..a229d1eb11 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -32,6 +32,7 @@ class MockLocationsCatalog implements LocationsCatalog { removeLocation = jest.fn(); locations = jest.fn(); location = jest.fn(); + locationHistory = jest.fn(); } describe('createRouter', () => { From 6b2be283a04719522e8194b257668f8c830852a8 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Thu, 28 May 2020 11:46:51 +0200 Subject: [PATCH 4/5] fix: PR edits --- .../src/catalog/DatabaseLocationsCatalog.ts | 10 ++-- plugins/catalog-backend/src/catalog/types.ts | 16 +++--- .../src/database/Database.test.ts | 1 - .../catalog-backend/src/database/Database.ts | 51 ++++++++----------- ...7114117_location_update_log_latest_view.ts | 35 +++++++++++++ plugins/catalog-backend/src/database/types.ts | 6 +-- .../catalog-backend/src/ingestion/types.ts | 2 +- 7 files changed, 72 insertions(+), 49 deletions(-) create mode 100644 plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index d79769a5a7..93a4a77622 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -18,7 +18,7 @@ import { LocationReader } from '../ingestion'; import { Database, DatabaseLocationUpdateLogEvent } from '../database'; import { AddLocation, - LocationEnvelope, + LocationResponse, Location, LocationsCatalog, } from './types'; @@ -47,10 +47,10 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { await this.database.removeLocation(id); } - async locations(): Promise { + async locations(): Promise { const items = await this.database.locations(); return items.map(({ message, status, timestamp, ...data }) => ({ - lastUpdate: { + currentStatus: { message, status, timestamp, @@ -63,7 +63,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { return this.database.locationHistory(id); } - async location(id: string): Promise { + async location(id: string): Promise { const { message, status, @@ -71,7 +71,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { ...data } = await this.database.location(id); return { - lastUpdate: { + currentStatus: { message, status, timestamp, diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 1c51448f0b..32d4edd27e 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -42,10 +42,10 @@ export type EntitiesCatalog = { // Locations // -export type status = { - timestamp: DatabaseLocationUpdateLogEvent['created_at'] | null; - status: DatabaseLocationUpdateLogEvent['status'] | null; - message: DatabaseLocationUpdateLogEvent['message'] | null; +export type LocationUpdateStatus = { + timestamp: string | null; + status: string | null; + message: string | null; }; export type Location = { @@ -54,9 +54,9 @@ export type Location = { target: string; }; -export type LocationEnvelope = { +export type LocationResponse = { data: Location; - lastUpdate: status; + currentStatus: LocationUpdateStatus; }; export type AddLocation = { @@ -74,7 +74,7 @@ export const addLocationSchema: yup.Schema = yup export type LocationsCatalog = { addLocation(location: AddLocation): Promise; removeLocation(id: string): Promise; - locations(): Promise; - location(id: string): Promise; + locations(): Promise; + location(id: string): Promise; locationHistory(id: string): Promise; }; diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index 070e138258..00be7d5d1e 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -78,7 +78,6 @@ describe('Database', () => { labels: { e: 'f' }, annotations: { g: 'h', - 'backstage.io/managed-by-location': undefined, }, }, spec: { i: 'j' }, diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index 674624c9c6..d4d3ec842e 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -195,7 +195,9 @@ export class Database { generation: 1, annotations: { ...(newEntity.metadata?.annotations ?? {}), - 'backstage.io/managed-by-location': request.locationId, + ...(request.locationId + ? { 'backstage.io/managed-by-location': request.locationId } + : {}), }, }; @@ -396,19 +398,15 @@ export class Database { async location(id: string): Promise { const items = await this.database('locations') .where('locations.id', id) - .leftJoin( - 'location_update_log', + .leftOuterJoin( + 'location_update_log_latest', 'locations.id', - 'location_update_log.location_id', + 'location_update_log_latest.location_id', ) - .orderBy('location_update_log.created_at', 'desc') - .select({ - status: 'location_update_log.status', - timestamp: 'location_update_log.created_at', - message: 'location_update_log.message', - id: 'locations.id', - type: 'locations.type', - target: 'locations.target', + .select('locations.*', { + status: 'location_update_log_latest.status', + timestamp: 'location_update_log_latest.created_at', + message: 'location_update_log_latest.message', }); if (!items.length) { @@ -418,28 +416,19 @@ export class Database { } async locations(): Promise { - const query = this.database - .select({ - status: 'location_update_log.status', - timestamp: 'location_update_log.created_at', - message: 'location_update_log.message', - id: 'locations.id', - type: 'locations.type', - target: 'locations.target', - }) - .from('locations') - .leftJoin( - 'location_update_log', + const locations = await this.database('locations') + .leftOuterJoin( + 'location_update_log_latest', 'locations.id', - 'location_update_log.location_id', + 'location_update_log_latest.location_id', ) - .orderBy('location_update_log.created_at', 'desc'); + .select('locations.*', { + status: 'location_update_log_latest.status', + timestamp: 'location_update_log_latest.created_at', + message: 'location_update_log_latest.message', + }); - const result = await this.database(query) - .select() - .groupBy('id'); - - return result; + return locations; } async locationHistory(id: string): Promise { diff --git a/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts b/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts new file mode 100644 index 0000000000..6da9eeed9b --- /dev/null +++ b/plugins/catalog-backend/src/database/migrations/20200527114117_location_update_log_latest_view.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + // Need to first order by date of creation + const query = knex + .select() + .from('location_update_log') + .orderBy('location_update_log.created_at', 'desc'); + + // And only then to do the grouping to get the latest per location + const groupedQuery = knex(query).groupBy('location_id').select(); + + await knex.schema.raw( + `CREATE VIEW location_update_log_latest AS ${groupedQuery.toString()};`, + ); +} + +export async function down(knex: Knex): Promise { + return knex.schema.raw(`DROP VIEW location_update_log_latest;`); +} diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index e20ba59517..cf49adb32e 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -53,9 +53,9 @@ export type DbLocationsRow = { }; export type DbLocationsRowWithStatus = DbLocationsRow & { - status: DatabaseLocationUpdateLogEvent['status'] | null; - timestamp: DatabaseLocationUpdateLogEvent['created_at'] | null; - message: DatabaseLocationUpdateLogEvent['message'] | null; + status: string | null; + timestamp: string | null; + message: string | null; }; export type AddDatabaseLocation = { diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 87a6f19f1f..e5083a65f1 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -79,7 +79,7 @@ export type EntityMeta = { * Key/value pairs of non-identifying auxiliary information attached to the * entity. */ - annotations?: Record; + annotations?: Record; }; /** From 3673722f69a1ec1fbe81b8e20b6386ca1f6fec6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 11:52:50 +0200 Subject: [PATCH 5/5] Move the model into a plugin-catalog-model for sharing outside the backend --- packages/backend/package.json | 13 +- packages/backend/src/plugins/catalog.ts | 14 +- packages/catalog-model/.eslintrc.js | 3 + packages/catalog-model/README.md | 12 + packages/catalog-model/package.json | 34 +++ packages/catalog-model/src/EntityPolicies.ts | 88 ++++++++ packages/catalog-model/src/entity/Entity.ts | 108 +++++++++ .../catalog-model/src/entity}/index.ts | 6 +- .../policies/FieldFormatEntityPolicy.test.ts | 105 +++++++++ .../policies/FieldFormatEntityPolicy.ts | 91 ++++++++ .../ForeignRootFieldsEntityPolicy.test.ts | 52 +++++ .../policies/ForeignRootFieldsEntityPolicy.ts | 40 ++++ .../ReservedFieldsEntityPolicy.test.ts | 62 ++++++ .../policies/ReservedFieldsEntityPolicy.ts | 66 ++++++ .../policies/SchemaValidEntityPolicy.test.ts | 176 +++++++++++++++ .../policies/SchemaValidEntityPolicy.ts | 80 +++++++ .../src/entity/policies/index.ts | 20 ++ packages/catalog-model/src/index.ts | 21 ++ .../src/kinds/ComponentV1beta1.ts | 32 +-- packages/catalog-model/src/kinds/index.ts | 20 ++ packages/catalog-model/src/setupTests.ts | 15 ++ packages/catalog-model/src/types.ts | 32 +++ .../CommonValidatorFunctions.test.ts | 0 .../validation/CommonValidatorFunctions.ts | 0 .../KubernetesValidatorFunctions.test.ts | 0 .../KubernetesValidatorFunctions.ts | 0 .../catalog-model/src/validation/index.ts | 20 ++ .../src/validation/makeValidator.ts | 0 .../catalog-model}/src/validation/types.ts | 0 plugins/catalog-backend/package.json | 5 +- .../src/catalog/DatabaseEntitiesCatalog.ts | 8 +- .../catalog/DatabaseLocationsCatalog.test.ts | 40 ++-- .../src/catalog/DatabaseLocationsCatalog.ts | 14 +- .../src/catalog/StaticEntitiesCatalog.ts | 12 +- plugins/catalog-backend/src/catalog/types.ts | 8 +- .../src/database/Database.test.ts | 10 +- .../catalog-backend/src/database/Database.ts | 14 +- .../src/database/DatabaseManager.test.ts | 82 ++++--- .../src/database/DatabaseManager.ts | 32 +-- ...0200520140700_location_update_log_table.ts | 5 +- .../src/database/search.test.ts | 6 +- .../catalog-backend/src/database/search.ts | 4 +- plugins/catalog-backend/src/database/types.ts | 6 +- .../src/ingestion/DescriptorParsers.ts | 53 ----- .../src/ingestion/IngestionModels.ts | 73 +++++++ .../src/ingestion/LocationReaders.ts | 39 ---- .../ingestion/descriptor/DescriptorParsers.ts | 45 ++++ .../src/ingestion/descriptor/index.ts | 18 ++ .../parsers/YamlDescriptorParser.ts | 64 ++++++ .../src/ingestion/descriptor/parsers/types.ts | 42 ++++ .../DescriptorEnvelopeParser.test.ts | 172 --------------- .../descriptors/DescriptorEnvelopeParser.ts | 206 ------------------ .../catalog-backend/src/ingestion/index.ts | 7 +- .../src/ingestion/source/LocationReaders.ts | 41 ++++ .../src/ingestion/source/index.ts | 20 ++ .../readers/FileLocationReader.ts} | 23 +- .../readers/GitHubLocationReader.test.ts} | 52 ++--- .../readers/GitHubLocationReader.ts} | 47 ++-- .../src/ingestion/source/readers/types.ts | 29 +++ .../src/ingestion/sources/util.ts | 55 ----- .../catalog-backend/src/ingestion/types.ts | 170 +-------------- .../src/service/router.test.ts | 8 +- plugins/catalog-backend/src/service/router.ts | 2 +- yarn.lock | 5 - 64 files changed, 1602 insertions(+), 925 deletions(-) create mode 100644 packages/catalog-model/.eslintrc.js create mode 100644 packages/catalog-model/README.md create mode 100644 packages/catalog-model/package.json create mode 100644 packages/catalog-model/src/EntityPolicies.ts create mode 100644 packages/catalog-model/src/entity/Entity.ts rename {plugins/catalog-backend/src/validation => packages/catalog-model/src/entity}/index.ts (79%) create mode 100644 packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/index.ts create mode 100644 packages/catalog-model/src/index.ts rename plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts => packages/catalog-model/src/kinds/ComponentV1beta1.ts (62%) create mode 100644 packages/catalog-model/src/kinds/index.ts create mode 100644 packages/catalog-model/src/setupTests.ts create mode 100644 packages/catalog-model/src/types.ts rename {plugins/catalog-backend => packages/catalog-model}/src/validation/CommonValidatorFunctions.test.ts (100%) rename {plugins/catalog-backend => packages/catalog-model}/src/validation/CommonValidatorFunctions.ts (100%) rename {plugins/catalog-backend => packages/catalog-model}/src/validation/KubernetesValidatorFunctions.test.ts (100%) rename {plugins/catalog-backend => packages/catalog-model}/src/validation/KubernetesValidatorFunctions.ts (100%) create mode 100644 packages/catalog-model/src/validation/index.ts rename {plugins/catalog-backend => packages/catalog-model}/src/validation/makeValidator.ts (100%) rename {plugins/catalog-backend => packages/catalog-model}/src/validation/types.ts (100%) delete mode 100644 plugins/catalog-backend/src/ingestion/DescriptorParsers.ts create mode 100644 plugins/catalog-backend/src/ingestion/IngestionModels.ts delete mode 100644 plugins/catalog-backend/src/ingestion/LocationReaders.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts delete mode 100644 plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/LocationReaders.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/index.ts rename plugins/catalog-backend/src/ingestion/{sources/FileLocationSource.ts => source/readers/FileLocationReader.ts} (63%) rename plugins/catalog-backend/src/ingestion/{sources/__tests__/GitHubLocationSource.test.ts => source/readers/GitHubLocationReader.test.ts} (64%) rename plugins/catalog-backend/src/ingestion/{sources/GitHubLocationSource.ts => source/readers/GitHubLocationReader.ts} (70%) create mode 100644 plugins/catalog-backend/src/ingestion/source/readers/types.ts delete mode 100644 plugins/catalog-backend/src/ingestion/sources/util.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 9aedafecbb..51fe04e51d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,7 +10,7 @@ }, "scripts": { "build": "tsc", - "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess nodemon", + "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"nodemon -r esm\\\"", "lint": "backstage-cli lint", "test": "backstage-cli test", "clean": "backstage-cli clean", @@ -18,13 +18,15 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@backstage/catalog-model": "^0.1.1-alpha.6", "@backstage/plugin-auth-backend": "^0.1.1-alpha.6", "@backstage/plugin-catalog-backend": "^0.1.1-alpha.6", - "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", - "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", "@backstage/plugin-identity-backend": "^0.1.1-alpha.6", + "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", + "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", + "esm": "^3.2.25", "express": "^4.17.1", "helmet": "^3.22.0", "knex": "^0.21.1", @@ -43,6 +45,9 @@ "typescript": "^3.9.2" }, "nodemonConfig": { - "watch": "./dist" + "watch": [ + "./dist", + "node_modules/@backstage*" + ] } } diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 687fd9157a..7e843cc80b 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -21,22 +21,28 @@ import { DatabaseManager, DescriptorParsers, LocationReaders, + IngestionModels, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; +import { EntityPolicies } from '@backstage/catalog-model'; export default async function ({ logger, database }: PluginEnvironment) { - const reader = LocationReaders.create(); - const parser = DescriptorParsers.create(); + const policy = new EntityPolicies(); + const ingestion = new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); const db = await DatabaseManager.createDatabase(database, logger); runPeriodically( - () => DatabaseManager.refreshLocations(db, reader, parser, logger), + () => DatabaseManager.refreshLocations(db, ingestion, policy, logger), 10000, ); const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db, reader); + const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion); return await createRouter({ entitiesCatalog, locationsCatalog, logger }); } diff --git a/packages/catalog-model/.eslintrc.js b/packages/catalog-model/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/packages/catalog-model/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/packages/catalog-model/README.md b/packages/catalog-model/README.md new file mode 100644 index 0000000000..755b9ee63c --- /dev/null +++ b/packages/catalog-model/README.md @@ -0,0 +1,12 @@ +# Catalog Model + +Contains the core model types and validators/policies used by the Backstage catalog functionality. + +This package will be imported both by the frontend and backend parts of the catalog, +as well as by others that want to consume catalog data. + +## Links + +- (Default frontend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog] +- (Default backend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog-backend] +- (The Backstage homepage)[https://backstage.io] diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json new file mode 100644 index 0000000000..ffd8fdba22 --- /dev/null +++ b/packages/catalog-model/package.json @@ -0,0 +1,34 @@ +{ + "name": "@backstage/catalog-model", + "version": "0.1.1-alpha.6", + "main": "dist/index.esm.js", + "main:src": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "lodash": "^4.17.15", + "yup": "^0.28.5" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.6", + "@types/jest": "^25.2.2", + "@types/lodash": "^4.14.151", + "@types/yup": "^0.28.2", + "yaml": "^1.9.2" + }, + "files": [ + "dist/**/*.{js,d.ts}" + ] +} diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts new file mode 100644 index 0000000000..cedf0df13d --- /dev/null +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + FieldFormatEntityPolicy, + ForeignRootFieldsEntityPolicy, + ReservedFieldsEntityPolicy, + SchemaValidEntityPolicy, +} from './entity'; +import { ComponentV1beta1Policy } from './kinds'; +import { EntityPolicy } from './types'; + +// Helper that requires that all of a set of policies can be successfully +// applied +class AllEntityPolicies implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + let result = entity; + for (const policy of this.policies) { + result = await policy.apply(entity); + } + return result; + } +} + +// Helper that requires that at least one of a set of policies can be +// successfully applied +class AnyEntityPolicy implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + for (const policy of this.policies) { + try { + return await policy.apply(entity); + } catch { + continue; + } + } + throw new Error(`The entity did not match any known policy`); + } +} + +export class EntityPolicies implements EntityPolicy { + private readonly policy: EntityPolicy; + + static defaultPolicies(): EntityPolicy { + return EntityPolicies.allOf([ + EntityPolicies.allOf([ + new SchemaValidEntityPolicy(), + new ForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy(), + new ReservedFieldsEntityPolicy(), + ]), + EntityPolicies.anyOf([new ComponentV1beta1Policy()]), + ]); + } + + static allOf(policies: EntityPolicy[]): EntityPolicy { + return new AllEntityPolicies(policies); + } + + static anyOf(policies: EntityPolicy[]): EntityPolicy { + return new AnyEntityPolicy(policies); + } + + constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) { + this.policy = policy; + } + + apply(entity: Entity): Promise { + return this.policy.apply(entity); + } +} diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts new file mode 100644 index 0000000000..6f57626749 --- /dev/null +++ b/packages/catalog-model/src/entity/Entity.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The format envelope that's common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type Entity = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Optional metadata related to the entity. + */ + metadata?: EntityMeta; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Metadata fields common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type EntityMeta = { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace, for any given kind. + */ + name?: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: Record; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: Record; +}; diff --git a/plugins/catalog-backend/src/validation/index.ts b/packages/catalog-model/src/entity/index.ts similarity index 79% rename from plugins/catalog-backend/src/validation/index.ts rename to packages/catalog-model/src/entity/index.ts index be607e43ec..9e96021336 100644 --- a/plugins/catalog-backend/src/validation/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -14,7 +14,5 @@ * limitations under the License. */ -export * from './CommonValidatorFunctions'; -export * from './KubernetesValidatorFunctions'; -export * from './makeValidator'; -export * from './types'; +export type { Entity, EntityMeta } from './Entity'; +export * from './policies'; diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts new file mode 100644 index 0000000000..d81b1155be --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; + +describe('FieldFormatEntityPolicy', () => { + let data: any; + let policy: FieldFormatEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new FieldFormatEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad apiVersion', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + data.apiVersion = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad kind', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + data.kind = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('handles missing metadata gracefully', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('handles missing spec gracefully', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name.*7/); + data.metadata.name = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/name.*aaaa/); + }); + + it('rejects bad namespace', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace.*7/); + data.metadata.namespace = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/namespace.*aaaa/); + }); + + it('rejects bad label key', async () => { + data.metadata.labels['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad label value', async () => { + data.metadata.labels.a = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad annotation key', async () => { + data.metadata.annotations['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*a#b/i); + }); + + it('rejects bad annotation value', async () => { + data.metadata.annotations.a = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*7/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts new file mode 100644 index 0000000000..1f94354f3f --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { makeValidator, Validators } from '../../validation'; +import { Entity } from '../Entity'; + +/** + * Ensures that the format of individual fields of the entity envelope + * is valid. + * + * This does not take into account machine generated fields such as uid, etag + * and generation. + */ +export class FieldFormatEntityPolicy implements EntityPolicy { + private readonly validators: Validators; + + constructor(validators: Validators = makeValidator()) { + this.validators = validators; + } + + async apply(entity: Entity): Promise { + function require( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + if (value === undefined || value === null) { + throw new Error(`${field} must have a value`); + } + + let isValid: boolean; + try { + isValid = validator(value); + } catch (e) { + throw new Error(`${field} could not be validated, ${e}`); + } + + if (!isValid) { + throw new Error(`${field} "${value}" is not valid`); + } + } + + function optional( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + return value === undefined || require(field, value, validator); + } + + require('apiVersion', entity.apiVersion, this.validators.isValidApiVersion); + require('kind', entity.kind, this.validators.isValidKind); + + optional( + 'metadata.name', + entity.metadata?.name, + this.validators.isValidEntityName, + ); + optional( + 'metadata.namespace', + entity.metadata?.namespace, + this.validators.isValidNamespace, + ); + + for (const [k, v] of Object.entries(entity.metadata?.labels ?? [])) { + require(`labels.${k}`, k, this.validators.isValidLabelKey); + require(`labels.${k}`, v, this.validators.isValidLabelValue); + } + + for (const [k, v] of Object.entries(entity.metadata?.annotations ?? [])) { + require(`annotations.${k}`, k, this.validators.isValidAnnotationKey); + require(`annotations.${k}`, v, this.validators.isValidAnnotationValue); + } + + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..98299259f8 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; + +describe('ForeignRootFieldsEntityPolicy', () => { + let data: any; + let policy: ForeignRootFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ForeignRootFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects unknown root fields', async () => { + data.spec2 = {}; + await expect(policy.apply(data)).rejects.toThrow(/spec2/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts new file mode 100644 index 0000000000..a4733e9a42 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec']; + +/** + * Ensures that there are no foreign root fields in the entity. + */ +export class ForeignRootFieldsEntityPolicy implements EntityPolicy { + private readonly knownFields: string[]; + + constructor(knownFields: string[] = defaultKnownFields) { + this.knownFields = knownFields; + } + + async apply(entity: Entity): Promise { + for (const field of Object.keys(entity)) { + if (!this.knownFields.includes(field)) { + throw new Error(`Unknown field ${field}`); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..348eabbdac --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; + +describe('ReservedFieldsEntityPolicy', () => { + let data: any; + let policy: ReservedFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ReservedFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects reserved keys in the spec root', async () => { + data.spec.apiVersion = 'a/b'; + await expect(policy.apply(data)).rejects.toThrow(/spec.*apiVersion/i); + }); + + it('rejects reserved keys in labels', async () => { + data.metadata.labels.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/label.*apiVersion/i); + }); + + it('rejects reserved keys in annotations', async () => { + data.metadata.annotations.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*apiVersion/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts new file mode 100644 index 0000000000..be2f732ca4 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_RESERVED_ENTITY_FIELDS = [ + 'apiVersion', + 'kind', + 'uid', + 'etag', + 'generation', + 'name', + 'namespace', + 'labels', + 'annotations', + 'spec', +]; + +/** + * Ensures that fields are not given certain reserved names. + */ +export class ReservedFieldsEntityPolicy implements EntityPolicy { + private readonly reservedFields: string[]; + + constructor(fields?: string[]) { + this.reservedFields = [ + ...(fields ?? []), + ...DEFAULT_RESERVED_ENTITY_FIELDS, + ]; + } + + async apply(entity: Entity): Promise { + for (const field of this.reservedFields) { + if (entity.spec?.hasOwnProperty(field)) { + throw new Error( + `The spec may not contain the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.labels?.hasOwnProperty(field)) { + throw new Error( + `A label may not have the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.annotations?.hasOwnProperty(field)) { + throw new Error( + `An annotation may not have the field ${field}, because it has reserved meaning`, + ); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts new file mode 100644 index 0000000000..b9d1165a60 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { Entity } from '../Entity'; +import { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; + +describe('SchemaValidEntityPolicy', () => { + let data: any; + let policy: SchemaValidEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new SchemaValidEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + // + // apiVersion and kind + // + + it('rejects wrong root type', async () => { + await expect(policy.apply((7 as unknown) as Entity)).rejects.toThrow( + /object/, + ); + }); + + it('rejects missing apiVersion', async () => { + delete data.apiVersion; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad apiVersion type', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects missing kind', async () => { + delete data.kind; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('rejects bad kind type', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + // + // metadata + // + + it('accepts missing metadata', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad metadata type', async () => { + data.metadata = 7; + await expect(policy.apply(data)).rejects.toThrow(/metadata/); + }); + + it('accepts missing uid', async () => { + delete data.metadata.uid; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad uid type', async () => { + data.metadata.uid = 7; + await expect(policy.apply(data)).rejects.toThrow(/uid/); + }); + + it('accepts missing etag', async () => { + delete data.metadata.etag; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad etag type', async () => { + data.metadata.etag = 7; + await expect(policy.apply(data)).rejects.toThrow(/etag/); + }); + + it('accepts missing generation', async () => { + delete data.metadata.generation; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad generation type', async () => { + data.metadata.generation = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/generation/); + }); + + it('accepts missing name', async () => { + delete data.metadata.name; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name type', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name/); + }); + + it('accepts missing namespace', async () => { + delete data.metadata.namespace; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad namespace type', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace/); + }); + + it('accepts missing labels', async () => { + delete data.metadata.labels; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad labels type', async () => { + data.metadata.labels = 7; + await expect(policy.apply(data)).rejects.toThrow(/labels/); + }); + + it('accepts missing annotations', async () => { + delete data.metadata.annotations; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad annotations type', async () => { + data.metadata.annotations = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotations/); + }); + + // + // spec + // + + it('accepts missing spec', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects non-object spec', async () => { + data.spec = 7; + await expect(policy.apply(data)).rejects.toThrow(/spec/); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts new file mode 100644 index 0000000000..7c0f5c20b6 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_ENTITY_SCHEMA = yup.object({ + apiVersion: yup.string().required(), + kind: yup.string().required(), + metadata: yup + .object({ + uid: yup + .string() + .notRequired() + .test( + 'metadata.uid', + 'The uid must not be empty', + value => value === undefined || value.length > 0, + ), + etag: yup + .string() + .notRequired() + .test( + 'metadata.etag', + 'The etag must not be empty', + value => value === undefined || value.length > 0, + ), + generation: yup + .number() + .notRequired() + .test( + 'metadata.generation', + 'The generation must be an integer greater than zero', + value => value === undefined || (value === (value | 0) && value > 0), + ), + name: yup.string().notRequired(), + namespace: yup.string().notRequired(), + labels: yup.object>().notRequired(), + annotations: yup.object>().notRequired(), + }) + .notRequired(), + spec: yup.object({}).notRequired(), +}); + +/** + * Ensures that the entity spec is valid according to a schema. + * + * This should be the first policy in the list, to ensure that other downstream + * policies can work with a structure that is at least valid in therms of the + * typescript type. + */ +export class SchemaValidEntityPolicy implements EntityPolicy { + private readonly schema: yup.Schema; + + constructor(schema: yup.Schema = DEFAULT_ENTITY_SCHEMA) { + this.schema = schema; + } + + async apply(entity: Entity): Promise { + try { + return await this.schema.validate(entity, { strict: true }); + } catch (e) { + throw new Error(`Malformed envelope, ${e}`); + } + } +} diff --git a/packages/catalog-model/src/entity/policies/index.ts b/packages/catalog-model/src/entity/policies/index.ts new file mode 100644 index 0000000000..f43aa68049 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; +export { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; +export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; +export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts new file mode 100644 index 0000000000..fb51461053 --- /dev/null +++ b/packages/catalog-model/src/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './entity'; +export { EntityPolicies } from './EntityPolicies'; +export * from './kinds'; +export type { EntityPolicy } from './types'; +export * from './validation'; diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/packages/catalog-model/src/kinds/ComponentV1beta1.ts similarity index 62% rename from plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts rename to packages/catalog-model/src/kinds/ComponentV1beta1.ts index 974b34aa8e..b0a3f627a5 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ b/packages/catalog-model/src/kinds/ComponentV1beta1.ts @@ -15,19 +15,28 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope, KindParser, ParserError } from '../types'; +import type { Entity, EntityMeta } from '../entity/Entity'; +import type { EntityPolicy } from '../types'; -export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope { +const API_VERSION = 'backstage.io/v1beta1'; +const KIND = 'Component'; + +export interface ComponentV1beta1 extends Entity { + apiVersion: typeof API_VERSION; + kind: typeof KIND; + metadata: EntityMeta & { + name: string; + }; spec: { type: string; }; } -export class ComponentDescriptorV1beta1Parser implements KindParser { +export class ComponentV1beta1Policy implements EntityPolicy { private schema: yup.Schema; constructor() { - this.schema = yup.object>({ + this.schema = yup.object>({ metadata: yup .object({ name: yup.string().required(), @@ -41,23 +50,14 @@ export class ComponentDescriptorV1beta1Parser implements KindParser { }); } - async tryParse( - envelope: DescriptorEnvelope, - ): Promise { + async apply(envelope: Entity): Promise { if ( envelope.apiVersion !== 'backstage.io/v1beta1' || envelope.kind !== 'Component' ) { - return undefined; + throw new Error('Unsupported apiVersion / kind'); } - try { - return await this.schema.validate(envelope, { strict: true }); - } catch (e) { - throw new ParserError( - `Malformed component, ${e}`, - envelope.metadata?.name, - ); - } + return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts new file mode 100644 index 0000000000..97d22c14a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ComponentV1beta1 } from './ComponentV1beta1'; +export { ComponentV1beta1Policy } from './ComponentV1beta1'; +export { ComponentV1beta1 as Component }; +export { ComponentV1beta1 }; diff --git a/packages/catalog-model/src/setupTests.ts b/packages/catalog-model/src/setupTests.ts new file mode 100644 index 0000000000..f3b69cc361 --- /dev/null +++ b/packages/catalog-model/src/setupTests.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts new file mode 100644 index 0000000000..1d581cf23f --- /dev/null +++ b/packages/catalog-model/src/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from './entity/Entity'; + +/** + * A policy for validation or mutation to be applied to entities as they are + * entering the system. + */ +export type EntityPolicy = { + /** + * Applies validation or mutation on an entity. + * + * @param entity The entity, as validated/mutated so far in the policy tree + * @returns The incoming entity, or a mutated version of the same + * @throws An error if the entity should be rejected + */ + apply(entity: Entity): Promise; +}; diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts similarity index 100% rename from plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts rename to packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts similarity index 100% rename from plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts rename to packages/catalog-model/src/validation/CommonValidatorFunctions.ts diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts similarity index 100% rename from plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts rename to packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts similarity index 100% rename from plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts rename to packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts diff --git a/packages/catalog-model/src/validation/index.ts b/packages/catalog-model/src/validation/index.ts new file mode 100644 index 0000000000..d679a5323c --- /dev/null +++ b/packages/catalog-model/src/validation/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CommonValidatorFunctions } from './CommonValidatorFunctions'; +export { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +export { makeValidator } from './makeValidator'; +export type { Validators } from './types'; diff --git a/plugins/catalog-backend/src/validation/makeValidator.ts b/packages/catalog-model/src/validation/makeValidator.ts similarity index 100% rename from plugins/catalog-backend/src/validation/makeValidator.ts rename to packages/catalog-model/src/validation/makeValidator.ts diff --git a/plugins/catalog-backend/src/validation/types.ts b/packages/catalog-model/src/validation/types.ts similarity index 100% rename from plugins/catalog-backend/src/validation/types.ts rename to packages/catalog-model/src/validation/types.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f32ffffb6c..5ec37295ed 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,8 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", - "@types/node-fetch": "^2.5.7", - "@types/supertest": "^2.0.8", + "@backstage/catalog-model": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -38,6 +37,8 @@ "devDependencies": { "@backstage/cli": "^0.1.1-alpha.6", "@types/lodash": "^4.14.151", + "@types/node-fetch": "^2.5.7", + "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 9d0d8fcaf7..972410a639 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,21 +14,21 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { Database } from '../database'; -import { DescriptorEnvelope } from '../ingestion/types'; import { EntitiesCatalog, EntityFilters } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { constructor(private readonly database: Database) {} - async entities(filters?: EntityFilters): Promise { + async entities(filters?: EntityFilters): Promise { const items = await this.database.transaction(tx => this.database.entities(tx, filters), ); return items.map(i => i.entity); } - async entityByUid(uid: string): Promise { + async entityByUid(uid: string): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [{ key: 'uid', values: [uid] }]), ); @@ -40,7 +40,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [ { key: 'kind', values: [kind] }, diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 0181b7fc28..56a3b3828f 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -13,13 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import knex from 'knex'; import path from 'path'; - import { Database } from '../database'; -import { ReaderOutput } from '../ingestion/types'; -import { getVoidLogger } from '@backstage/backend-common'; +import { IngestionModel } from '../ingestion/types'; +import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; + +class MockIngestionModel implements IngestionModel { + readLocation = jest.fn(async (type: string, target: string) => { + if (type !== 'valid_type') { + throw new Error(`Unknown location type ${type}`); + } + if (target === 'valid_target') { + return [{ type: 'data', data: {} as Entity } as const]; + } + throw new Error( + `Can't read location at ${target} with error: Something is broken`, + ); + }); +} describe('DatabaseLocationsCatalog', () => { const database = knex({ @@ -32,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => { }); let db: Database; let catalog: DatabaseLocationsCatalog; - - const mockLocationReader = { - read: async (type: string, target: string): Promise => { - if (type !== 'valid_type') { - throw new Error(`Unknown location type ${type}`); - } - if (target === 'valid_target') { - return Promise.resolve([{ type: 'data', data: {} }]); - } - throw new Error( - `Can't read location at ${target} with error: Something is broken`, - ); - }, - }; + let ingestionModel: IngestionModel; beforeEach(async () => { await database.migrate.latest({ @@ -53,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => { loadExtensions: ['.ts'], }); db = new Database(database, getVoidLogger()); - catalog = new DatabaseLocationsCatalog(db, mockLocationReader); + ingestionModel = new MockIngestionModel(); + catalog = new DatabaseLocationsCatalog(db, ingestionModel); }); it('resolves to location with id', async () => { diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 6e3c9ae306..b13d70de86 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -15,17 +15,25 @@ */ import { Database } from '../database'; +import { IngestionModel } from '../ingestion/types'; import { AddLocation, Location, LocationsCatalog } from './types'; -import { LocationReader } from '../ingestion'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( private readonly database: Database, - private readonly reader: LocationReader, + private readonly ingestionModel: IngestionModel, ) {} async addLocation(location: AddLocation): Promise { - const outputs = await this.reader.read(location.type, location.target); + const outputs = await this.ingestionModel.readLocation( + location.type, + location.target, + ); + if (!outputs) { + throw new Error( + `Unknown location type ${location.type} ${location.target}`, + ); + } outputs.forEach(output => { if (output.type === 'error') { throw new Error( diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 371cdf1f76..1de606d44d 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -15,22 +15,22 @@ */ import { NotFoundError } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import lodash from 'lodash'; -import { DescriptorEnvelope } from '../ingestion'; import { EntitiesCatalog } from './types'; export class StaticEntitiesCatalog implements EntitiesCatalog { - private _entities: DescriptorEnvelope[]; + private _entities: Entity[]; - constructor(entities: DescriptorEnvelope[]) { + constructor(entities: Entity[]) { this._entities = entities; } - async entities(): Promise { + async entities(): Promise { return lodash.cloneDeep(this._entities); } - async entityByUid(uid: string): Promise { + async entityByUid(uid: string): Promise { const item = this._entities.find(e => uid === e.metadata?.uid); if (!item) { throw new NotFoundError('Entity cannot be found'); @@ -42,7 +42,7 @@ export class StaticEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const item = this._entities.find( e => kind === e.kind && diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index d5627ea86a..2f8967cd1b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; // // Entities @@ -28,13 +28,13 @@ export type EntityFilter = { export type EntityFilters = EntityFilter[]; export type EntitiesCatalog = { - entities(filters?: EntityFilters): Promise; - entityByUid(uid: string): Promise; + entities(filters?: EntityFilters): Promise; + entityByUid(uid: string): Promise; entityByName( kind: string, namespace: string | undefined, name: string, - ): Promise; + ): Promise; }; // diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index a82385aae4..7a38adf02f 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -19,9 +19,9 @@ import { getVoidLogger, NotFoundError, } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import Knex from 'knex'; import path from 'path'; -import { DescriptorEnvelope } from '../ingestion'; import { Database } from './Database'; import { AddDatabaseLocation, @@ -247,8 +247,8 @@ describe('Database', () => { describe('entities', () => { it('can get all entities with empty filters list', async () => { const catalog = new Database(database, getVoidLogger()); - const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' }; - const e2: DescriptorEnvelope = { + const e1: Entity = { apiVersion: 'a', kind: 'b' }; + const e2: Entity = { apiVersion: 'a', kind: 'b', spec: { c: null }, @@ -271,7 +271,7 @@ describe('Database', () => { it('can get all specific entities for matching filters (naive case)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ + const entities: Entity[] = [ { apiVersion: 'a', kind: 'b' }, { apiVersion: 'a', @@ -305,7 +305,7 @@ describe('Database', () => { it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ + const entities: Entity[] = [ { apiVersion: 'a', kind: 'b' }, { apiVersion: 'a', diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index 614f7d09bf..db9e76a0a8 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -19,12 +19,12 @@ import { InputError, NotFoundError, } from '@backstage/backend-common'; +import { Entity, EntityMeta } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { EntityFilters } from '../catalog'; -import { DescriptorEnvelope, EntityMeta } from '../ingestion'; import { buildEntitySearch } from './search'; import { AddDatabaseLocation, @@ -54,9 +54,7 @@ function serializeMetadata(metadata: EntityMeta | undefined): string | null { return JSON.stringify(getStrippedMetadata(metadata)); } -function serializeSpec( - spec: DescriptorEnvelope['spec'], -): DbEntitiesRow['spec'] { +function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] { if (!spec) { return null; } @@ -66,7 +64,7 @@ function serializeSpec( function toEntityRow( locationId: string | undefined, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesRow { return { id: entity.metadata!.uid!, @@ -83,7 +81,7 @@ function toEntityRow( } function toEntityResponse(row: DbEntitiesRow): DbEntityResponse { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: row.api_version, kind: row.kind, metadata: { @@ -94,7 +92,7 @@ function toEntityResponse(row: DbEntitiesRow): DbEntityResponse { }; if (row.metadata) { - const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata']; + const metadata = JSON.parse(row.metadata) as Entity['metadata']; entity.metadata = { ...entity.metadata, ...metadata }; } @@ -422,7 +420,7 @@ export class Database { private async updateEntitiesSearch( tx: Knex.Transaction, entityId: string, - data: DescriptorEnvelope, + data: Entity, ): Promise { try { const entries = buildEntitySearch(entityId, data); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index ca50f518cb..c8d0aec332 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -15,13 +15,9 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; -import { - ComponentDescriptor, - DescriptorParser, - LocationReader, - ParserError, -} from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; @@ -32,18 +28,18 @@ describe('DatabaseManager', () => { const db = ({ locations: jest.fn().mockResolvedValue([]), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(), + const reader: IngestionModel = { + readLocation: jest.fn(), }; - const parser: DescriptorParser = { - parse: jest.fn(), + const policy: EntityPolicy = { + apply: jest.fn(), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).not.toHaveBeenCalled(); - expect(parser.parse).not.toHaveBeenCalled(); + expect(reader.readLocation).not.toHaveBeenCalled(); + expect(policy.apply).not.toHaveBeenCalled(); }); it('can update a single location', async () => { @@ -52,7 +48,7 @@ describe('DatabaseManager', () => { type: 'some', target: 'thing', }; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, @@ -68,18 +64,20 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as Partial) as Database; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).toHaveBeenCalledTimes(1); - expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing'); + expect(reader.readLocation).toHaveBeenCalledTimes(1); + expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing'); expect(db.addEntity).toHaveBeenCalledTimes(1); expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, { locationId: '123', @@ -108,21 +106,23 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -158,23 +158,23 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), - }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), ), }; + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.reject(new Error('parser error message'))), + }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -211,19 +211,17 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(() => + const reader: IngestionModel = { + readLocation: jest.fn(() => Promise.reject([{ type: 'error', error: new Error('test message') }]), ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), - ), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.reject(new Error('parser error message'))), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 7f646971db..dff589c525 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,18 +14,14 @@ * limitations under the License. */ +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; -import { - DescriptorEnvelope, - DescriptorParser, - LocationReader, - ParserError, -} from '../ingestion'; import { Database } from './Database'; import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types'; +import { IngestionModel } from '../ingestion/types'; export class DatabaseManager { public static async createDatabase( @@ -67,8 +63,8 @@ export class DatabaseManager { public static async refreshLocations( database: Database, - reader: LocationReader, - parser: DescriptorParser, + ingestionModel: IngestionModel, + entityPolicy: EntityPolicy, logger: Logger, ): Promise { const locations = await database.locations(); @@ -78,7 +74,10 @@ export class DatabaseManager { `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`, ); - const readerOutput = await reader.read(location.type, location.target); + const readerOutput = await ingestionModel.readLocation( + location.type, + location.target, + ); for (const readerItem of readerOutput) { if (readerItem.type === 'error') { @@ -87,7 +86,7 @@ export class DatabaseManager { } try { - const entity = await parser.parse(readerItem.data); + const entity = await entityPolicy.apply(readerItem.data); await DatabaseManager.refreshSingleEntity( database, location.id, @@ -100,15 +99,11 @@ export class DatabaseManager { entity.metadata!.name, ); } catch (error) { - let entityName; - if (error instanceof ParserError) { - entityName = error.entityName; - } await DatabaseManager.logUpdateFailure( database, location.id, error, - entityName, + readerItem.data.metadata?.name, ); } } @@ -129,7 +124,7 @@ export class DatabaseManager { private static async refreshSingleEntity( database: Database, locationId: string, - entity: DescriptorEnvelope, + entity: Entity, logger: Logger, ): Promise { const { kind } = entity; @@ -163,10 +158,7 @@ export class DatabaseManager { }); } - private static entitiesAreEqual( - first: DescriptorEnvelope, - second: DescriptorEnvelope, - ) { + private static entitiesAreEqual(first: Entity, second: Entity) { const firstClone = lodash.cloneDeep(first); const secondClone = lodash.cloneDeep(second); diff --git a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts index b2e1dc0d32..6700f5748e 100644 --- a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts +++ b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts @@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise { return knex.schema.createTable('location_update_log', table => { table.uuid('id').primary(); table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable(); table.string('message'); table .uuid('location_id') diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts index 26b20d1846..7ad06aee14 100644 --- a/plugins/catalog-backend/src/database/search.test.ts +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { buildEntitySearch, visitEntityPart } from './search'; import { DbEntitiesSearchRow } from './types'; @@ -99,7 +99,7 @@ describe('search', () => { describe('buildEntitySearch', () => { it('adds special keys even if missing', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', }; @@ -116,7 +116,7 @@ describe('search', () => { }); it('adds prefix-stripped versions', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', metadata: { diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts index f54cfde7e6..fcacf1a9f2 100644 --- a/plugins/catalog-backend/src/database/search.ts +++ b/plugins/catalog-backend/src/database/search.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { DbEntitiesSearchRow } from './types'; // Search entries that start with these prefixes, also get a shorthand without @@ -119,7 +119,7 @@ export function visitEntityPart( */ export function buildEntitySearch( entityId: string, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesSearchRow[] { // Start with some special keys that are always present because you want to // be able to easily search for null specifically diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 06e234545f..ca1cbbdfd4 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; export type DbEntitiesRow = { id: string; @@ -32,12 +32,12 @@ export type DbEntitiesRow = { export type DbEntityRequest = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntityResponse = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntitiesSearchRow = { diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts deleted file mode 100644 index 86283df666..0000000000 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { makeValidator } from '../validation'; -import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; -import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; -import { - DescriptorEnvelope, - DescriptorParser, - KindParser, - ParserError, -} from './types'; - -export class DescriptorParsers implements DescriptorParser { - static create(): DescriptorParser { - const validators = makeValidator(); - return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [ - new ComponentDescriptorV1beta1Parser(), - ]); - } - - constructor( - private readonly envelopeParser: DescriptorEnvelopeParser, - private readonly kindParsers: KindParser[], - ) {} - - async parse(descriptor: object): Promise { - const envelope = await this.envelopeParser.parse(descriptor); - for (const parser of this.kindParsers) { - const parsed = await parser.tryParse(envelope); - if (parsed) { - return parsed; - } - } - throw new ParserError( - `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`, - envelope.metadata?.name, - ); - } -} diff --git a/plugins/catalog-backend/src/ingestion/IngestionModels.ts b/plugins/catalog-backend/src/ingestion/IngestionModels.ts new file mode 100644 index 0000000000..616f1b49ae --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/IngestionModels.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model'; +import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types'; +import { LocationReader, LocationReaders } from './source'; +import { IngestionModel } from './types'; +import { DescriptorParsers } from './descriptor'; + +export class IngestionModels implements IngestionModel { + private readonly reader: LocationReader; + private readonly parser: DescriptorParser; + private readonly entityPolicy: EntityPolicy; + + static default(): IngestionModel { + return new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); + } + + constructor( + reader: LocationReader, + parser: DescriptorParser, + entityPolicy: EntityPolicy, + ) { + this.reader = reader; + this.parser = parser; + this.entityPolicy = entityPolicy; + } + + async readLocation(type: string, target: string): Promise { + const buffer = await this.reader.tryRead(type, target); + if (!buffer) { + throw new Error(`No reader could handle location ${type} ${target}`); + } + + const items = await this.parser.tryParse(buffer); + if (!items) { + throw new Error(`No parser could handle location ${type} ${target}`); + } + + const result: ReaderOutput[] = []; + for (const item of items) { + if (item.type === 'error') { + result.push(item); + } else { + try { + const output = await this.entityPolicy.apply(item.data); + result.push({ type: 'data', data: output }); + } catch (e) { + result.push({ type: 'error', error: e }); + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts deleted file mode 100644 index aa88d299e4..0000000000 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { FileLocationSource } from './sources/FileLocationSource'; -import { GitHubLocationSource } from './sources/GitHubLocationSource'; -import { LocationReader, LocationSource, ReaderOutput } from './types'; - -export class LocationReaders implements LocationReader { - static create(): LocationReader { - return new LocationReaders({ - file: new FileLocationSource(), - github: new GitHubLocationSource(), - }); - } - - constructor(private readonly sources: Record) {} - - async read(type: string, target: string): Promise { - const source = this.sources[type]; - if (!source) { - throw new Error(`Unknown location type ${type}`); - } - - return source.read(target); - } -} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts new file mode 100644 index 0000000000..ed05855109 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DescriptorParser, ReaderOutput } from './parsers/types'; +import { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; + +/** + * Parses raw descriptor data (e.g. from a file or stream) into entities. + */ +export class DescriptorParsers implements DescriptorParser { + private readonly parsers: DescriptorParser[]; + + static defaultParsers(): DescriptorParser[] { + return [new YamlDescriptorParser()]; + } + + constructor( + parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(), + ) { + this.parsers = parsers; + } + + async tryParse(data: Buffer): Promise { + for (const parser of this.parsers) { + const result = await parser.tryParse(data); + if (result) { + return result; + } + } + throw new Error(`Unsupported descriptor format`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/index.ts b/plugins/catalog-backend/src/ingestion/descriptor/index.ts new file mode 100644 index 0000000000..1529c78afc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DescriptorParsers } from './DescriptorParsers'; +export { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts new file mode 100644 index 0000000000..3f2e9e3c5c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import yaml from 'yaml'; +import { DescriptorParser, ReaderOutput } from './types'; + +/** + * Parses descriptors on YAML format + */ +export class YamlDescriptorParser implements DescriptorParser { + async tryParse(data: Buffer): Promise { + // TODO(freben): Should perhaps first do format detection, so the parse + // failure can be emitted as a proper error instead of just as if we + // weren't handling the format at all. + let documents; + try { + documents = yaml.parseAllDocuments(data.toString('utf8')); + } catch (e) { + return undefined; + } + + const result: ReaderOutput[] = []; + + for (const document of documents) { + if (document.contents) { + if (document.errors?.length) { + result.push({ + type: 'error', + error: new Error(`Malformed YAML document, ${document.errors[0]}`), + }); + } else { + const json = document.toJSON(); + if (typeof json !== 'object' || Array.isArray(json)) { + result.push({ + type: 'error', + error: new Error(`Malformed descriptor, expected object at root`), + }); + } else { + result.push({ + type: 'data', + data: json as Entity, + }); + } + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts new file mode 100644 index 0000000000..baef5dc3df --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; + +export type ReaderOutput = + | { type: 'error'; error: Error } + | { type: 'data'; data: Entity }; + +/** + * Parses raw descriptor data (e.g. from a file) into entities. + */ +export type DescriptorParser = { + /** + * Try to parse some raw data into an entity. + * + * Note that this is only the low level operation of parsing the raw file + * format, e.g. reading JSON or YAML or similar and emitting as structured + * but unvalidated data. The actual validation is performed by EntityPolicy + * and KindParser. + * + * @param data Raw descriptor data + * @returns A list of raw unvalidated entities / errors, or undefined if the + * given data is not meant to be handled by this parser + * @throws An Error if the format was handled and found to not be properly + * formed + */ + tryParse(data: Buffer): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts deleted file mode 100644 index 7c96fb7cf5..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { makeValidator } from '../../validation'; -import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser'; - -describe('DescriptorEnvelopeParser', () => { - let data: any; - let parser: DescriptorEnvelopeParser; - - beforeEach(() => { - data = yaml.parse(` - apiVersion: backstage.io/v1beta1 - kind: Component - metadata: - uid: e01199ab-08cc-44c2-8e19-5c29ded82521 - etag: lsndfkjsndfkjnsdfkjnsd== - generation: 13 - name: my-component-yay - namespace: the-namespace - labels: - backstage.io/custom: ValueStuff - annotations: - example.com/bindings: are-secret - spec: - custom: stuff - `); - parser = new DescriptorEnvelopeParser(makeValidator()); - }); - - it('works for the happy path', async () => { - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects missing apiVersion', async () => { - delete data.apiVersion; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects wrong root type', async () => { - await expect(parser.parse(7)).rejects.toThrow(/object/); - }); - - it('rejects bad apiVersion', async () => { - data.apiVersion = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects missing kind', async () => { - delete data.kind; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('rejects bad kind', async () => { - data.kind = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('accepts missing metadata', async () => { - delete data.medatata; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object metadata', async () => { - data.metadata = 7; - await expect(parser.parse(data)).rejects.toThrow(/metadata/); - }); - - it('accepts missing uid', async () => { - delete data.metadata.uid; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad uid', async () => { - data.metadata.uid = 7; - await expect(parser.parse(data)).rejects.toThrow(/uid/); - }); - - it('accepts missing etag', async () => { - delete data.metadata.etag; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad etag', async () => { - data.metadata.etag = 7; - await expect(parser.parse(data)).rejects.toThrow(/etag/); - }); - - it('accepts missing generation', async () => { - delete data.metadata.generation; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad generation', async () => { - data.metadata.generation = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/generation/); - }); - - it('accepts missing spec', async () => { - delete data.spec; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object spec', async () => { - data.spec = 7; - await expect(parser.parse(data)).rejects.toThrow(/spec/); - }); - - it('rejects bad name', async () => { - data.metadata.name = 7; - await expect(parser.parse(data)).rejects.toThrow(/name/); - }); - - it('rejects bad namespace', async () => { - data.metadata.namespace = 7; - await expect(parser.parse(data)).rejects.toThrow(/namespace/); - }); - - it('rejects bad label key', async () => { - data.metadata.labels['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/label.*key/i); - }); - - it('rejects bad label value', async () => { - data.metadata.labels.a = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/label.*value/i); - }); - - it('rejects bad annotation key', async () => { - data.metadata.annotations['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i); - }); - - it('rejects bad annotation value', async () => { - data.metadata.annotations.a = []; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i); - }); - - it('rejects unknown root keys', async () => { - data.spec2 = {}; - await expect(parser.parse(data)).rejects.toThrow(/spec2/i); - }); - - it('rejects reserved keys in the spec root', async () => { - data.spec.apiVersion = 'a/b'; - await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i); - }); - - it('rejects reserved keys in labels', async () => { - data.metadata.labels.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i); - }); - - it('rejects reserved keys in annotations', async () => { - data.metadata.annotations.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts deleted file mode 100644 index 1f21e2df9e..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as yup from 'yup'; -import { Validators } from '../../validation'; -import { DescriptorEnvelope } from '../types'; - -/** - * Parses some raw structured data as a descriptor envelope - */ -export class DescriptorEnvelopeParser { - private schema: yup.Schema; - - constructor(validators: Validators) { - const apiVersionSchema = yup - .string() - .required() - .test( - 'apiVersion', - 'The apiVersion is not formatted according to schema', - validators.isValidApiVersion, - ); - - const kindSchema = yup - .string() - .required() - .test( - 'kind', - 'The kind is not formatted according to schema', - validators.isValidKind, - ); - - const uidSchema = yup - .string() - .notRequired() - .test( - 'metadata.uid', - 'The uid is not formatted according to schema', - value => value === undefined || value.length > 0, - ); - - const etagSchema = yup - .string() - .notRequired() - .test( - 'metadata.etag', - 'The etag value is not according to schema', - value => value === undefined || value.length > 0, - ); - - const generationSchema = yup - .number() - .notRequired() - .test( - 'metadata.generation', - 'The generation value is not according to schema', - value => value === undefined || value > 0, - ); - - const nameSchema = yup - .string() - .notRequired() - .test( - 'metadata.name', - 'The name is not formatted according to schema', - value => value === undefined || validators.isValidEntityName(value), - ); - - const namespaceSchema = yup - .string() - .notRequired() - .test( - 'metadata.namespace', - 'The namespace is malformed', - value => value === undefined || validators.isValidNamespace(value), - ); - - const labelsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.labels.keys', - message: 'Label keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidLabelKey) - ); - }, - }) - .test({ - name: 'metadata.labels.values', - message: 'Label values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidLabelValue) - ); - }, - }); - - const annotationsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.annotations.keys', - message: 'Annotation keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidAnnotationKey) - ); - }, - }) - .test({ - name: 'metadata.annotations.values', - message: 'Annotation values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidAnnotationValue) - ); - }, - }); - - const metadataSchema = yup - .object({ - uid: uidSchema, - etag: etagSchema, - generation: generationSchema, - name: nameSchema, - namespace: namespaceSchema, - labels: labelsSchema, - annotations: annotationsSchema, - }) - .notRequired(); - - const specSchema = yup.object({}).notRequired(); - - this.schema = yup - .object({ - apiVersion: apiVersionSchema, - kind: kindSchema, - metadata: metadataSchema, - spec: specSchema, - }) - .noUnknown(); - } - - async parse(data: any): Promise { - let result: DescriptorEnvelope; - try { - result = await this.schema.validate(data, { strict: true }); - } catch (e) { - throw new Error(`Malformed envelope, ${e}`); - } - - // These are keys with specific semantic meaning in a document, that we do - // not want to appear in the root of the spec, or as labels or as - // annotations, because they will lead to confusion. - const reservedKeys = [ - 'apiVersion', - 'kind', - 'uid', - 'etag', - 'generation', - 'name', - 'namespace', - 'labels', - 'annotations', - 'spec', - ]; - for (const key of reservedKeys) { - if (result.spec?.hasOwnProperty(key)) { - throw new Error( - `The spec may not contain the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.labels?.hasOwnProperty(key)) { - throw new Error( - `A label may not have the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.annotations?.hasOwnProperty(key)) { - throw new Error( - `An annotation may not have the key ${key}, because it has reserved meaning`, - ); - } - } - - return result; - } -} diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index ca6f2dd2be..b6aceaecdd 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ -export * from './DescriptorParsers'; -export * from './LocationReaders'; -export * from './types'; +export * from './descriptor'; +export { IngestionModels } from './IngestionModels'; +export * from './source'; +export type { IngestionModel } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts new file mode 100644 index 0000000000..a670f309ad --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileLocationReader } from './readers/FileLocationReader'; +import { GitHubLocationReader } from './readers/GitHubLocationReader'; +import { LocationReader } from './readers/types'; + +export class LocationReaders implements LocationReader { + private readonly readers: LocationReader[]; + + static defaultReaders(): LocationReader[] { + return [new FileLocationReader(), new GitHubLocationReader()]; + } + + constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) { + this.readers = readers; + } + + async tryRead(type: string, target: string): Promise { + for (const reader of this.readers) { + const result = await reader.tryRead(type, target); + if (result) { + return result; + } + } + throw new Error(`Could not read unknown location "${type}", "${target}"`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/index.ts b/plugins/catalog-backend/src/ingestion/source/index.ts new file mode 100644 index 0000000000..3ed1063878 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { LocationReaders } from './LocationReaders'; +export { FileLocationReader } from './readers/FileLocationReader'; +export { GitHubLocationReader } from './readers/GitHubLocationReader'; +export { LocationReader } from './readers/types'; diff --git a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts similarity index 63% rename from plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts rename to plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts index 9d2794ec4f..0c64aebf14 100644 --- a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts @@ -15,22 +15,21 @@ */ import fs from 'fs-extra'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; +import { LocationReader } from './types'; + +/** + * Reads a file from the local file system. + */ +export class FileLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'file') { + return undefined; + } -export class FileLocationSource implements LocationSource { - async read(target: string): Promise { - let rawYaml; try { - rawYaml = await fs.readFile(target, 'utf8'); + return await fs.readFile(target); } catch (e) { throw new Error(`Unable to read "${target}", ${e}`); } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } } } diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts similarity index 64% rename from plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts rename to plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts index 083c2f7e86..1f0f6ed539 100644 --- a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts @@ -16,43 +16,26 @@ jest.mock('node-fetch'); -import fs from 'fs-extra'; import fetch from 'node-fetch'; -import path from 'path'; -import { GitHubLocationSource } from '../GitHubLocationSource'; +import { GitHubLocationReader } from './GitHubLocationReader'; const { Response } = jest.requireActual('node-fetch'); -const FIXTURES_DIR = path.resolve( - __dirname, - '..', - '..', - '..', - '..', - 'fixtures', -); -const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => { - acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8'); - return acc; -}, {} as Record); - -describe('Unit: GitHubLocationSource', () => { +describe('Unit: GitHubLocationReader', () => { beforeEach(() => { jest.clearAllMocks(); }); it('fetches the file and parses it correctly', async () => { - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures['one_component.yaml'])), - ); - const reader = new GitHubLocationSource(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); - const result = await reader.read( + const reader = new GitHubLocationReader(); + const buffer = await reader.tryRead( + 'github', 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', ); - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); + expect(buffer?.toString('utf8')).toBe('hello'); }); it('changes the url to point to https://raw.githubusercontent.com', async () => { @@ -61,12 +44,12 @@ describe('Unit: GitHubLocationSource', () => { const folderPath = `master/plugins/catalog-backend/fixtures`; const componentFilename = `one_component.yaml`; const rawGitHubUrl = `https://raw.githubusercontent.com`; - const reader = new GitHubLocationSource(); - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures[componentFilename])), - ); - await reader.read( + const reader = new GitHubLocationReader(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + await reader.tryRead( + 'github', `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, ); @@ -76,7 +59,7 @@ describe('Unit: GitHubLocationSource', () => { }); describe('rejects wrong urls', () => { - const reader = new GitHubLocationSource(); + const reader = new GitHubLocationReader(); it.each([ ['http://example.com/one_component.yaml'], @@ -87,7 +70,7 @@ describe('Unit: GitHubLocationSource', () => { ])( '%p', async (url: string) => - await expect(reader.read(url)).rejects.toThrow(/url/), + await expect(reader.tryRead('github', url)).rejects.toThrow(/url/), ); }); }); @@ -100,11 +83,10 @@ describe('Integration: GitHubLocationSource', () => { it('fetches the fixture from backstage repo', async () => { const PERMANENT_LINK = 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; - const reader = new GitHubLocationSource(); - const result = await reader.read(PERMANENT_LINK); + const reader = new GitHubLocationReader(); + const result = await reader.tryRead('github', PERMANENT_LINK); - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); + expect(result?.toString('utf8')).toContain('component3'); }); }); diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts similarity index 70% rename from plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts rename to plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts index 69ba4911a6..bd330e28b4 100644 --- a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts @@ -16,17 +16,31 @@ import fetch from 'node-fetch'; import { URL } from 'url'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; +import { LocationReader } from './types'; -// Pointing to raw.githubusercontent.com for now -// to be changed in the future, after auth and tokens are done -export class GitHubLocationSource implements LocationSource { - async read(target: string): Promise { - let url: URL; +/** + * Reads a file whose target is a GitHub URL. + * + * Uses raw.githubusercontent.com for now, but this will probably change in the + * future when token auth is implemented. + */ +export class GitHubLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'github') { + return undefined; + } + const url = this.buildRawUrl(target); try { - url = new URL(target); + return await fetch(url.toString()).then(x => x.buffer()); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } + + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); const [ empty, @@ -51,23 +65,10 @@ export class GitHubLocationSource implements LocationSource { url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); url.hostname = 'raw.githubusercontent.com'; url.protocol = 'https'; + + return url; } catch (e) { throw new Error(`Incorrect url: ${target}, ${e}`); } - - let rawYaml; - try { - rawYaml = await fetch(url.toString()).then(x => { - return x.text(); - }); - } catch (e) { - throw new Error(`Unable to read "${target}", ${e}`); - } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } } } diff --git a/plugins/catalog-backend/src/ingestion/source/readers/types.ts b/plugins/catalog-backend/src/ingestion/source/readers/types.ts new file mode 100644 index 0000000000..37c7b46885 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type LocationReader = { + /** + * Reads the contents of a single location. + * + * @param type The type of location to read + * @param target The location target (type-specific) + * @returns The target contents, as a raw Buffer, or undefined if this type + * was not meant to be consumed by this reader + * @throws An error if the type was meant for this reader, but could not be + * read + */ + tryRead(type: string, target: string): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/sources/util.ts b/plugins/catalog-backend/src/ingestion/sources/util.ts deleted file mode 100644 index cccbeb92b3..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/util.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { ReaderOutput } from '../types'; - -export function readDescriptorYaml(data: string): ReaderOutput[] { - let documents; - try { - documents = yaml.parseAllDocuments(data); - } catch (e) { - throw new Error(`Could not parse YAML data, ${e}`); - } - - const result: ReaderOutput[] = []; - - for (const document of documents) { - if (document.contents) { - if (document.errors?.length) { - result.push({ - type: 'error', - error: new Error(`Malformed YAML document, ${document.errors[0]}`), - }); - } else { - const json = document.toJSON(); - if (typeof json !== 'object' || Array.isArray(json)) { - result.push({ - type: 'error', - error: new Error(`Malformed descriptor, expected object at root`), - }); - } else { - result.push({ - type: 'data', - data: json, - }); - } - } - } - } - - return result; -} diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index e5083a65f1..8878c2af5b 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,172 +14,8 @@ * limitations under the License. */ -import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser'; +import { ReaderOutput } from './descriptor/parsers/types'; -export type ComponentDescriptor = ComponentDescriptorV1beta1; - -/** - * Metadata fields common to all versions/kinds of entity. - * - * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - */ -export type EntityMeta = { - /** - * A globally unique ID for the entity. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, but the server is free to reject requests - * that do so in such a way that it breaks semantics. - */ - uid?: string; - - /** - * An opaque string that changes for each update operation to any part of - * the entity, including metadata. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, and the server will then reject the - * operation if it does not match the current stored value. - */ - etag?: string; - - /** - * A positive nonzero number that indicates the current generation of data - * for this entity; the value is incremented each time the spec changes. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. - */ - generation?: number; - - /** - * The name of the entity. - * - * Must be uniqe within the catalog at any given point in time, for any - * given namespace, for any given kind. - */ - name?: string; - - /** - * The namespace that the entity belongs to. - */ - namespace?: string; - - /** - * Key/value pairs of identifying information attached to the entity. - */ - labels?: Record; - - /** - * Key/value pairs of non-identifying auxiliary information attached to the - * entity. - */ - annotations?: Record; -}; - -/** - * The format envelope that's common to all versions/kinds. - * - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ - */ -export type DescriptorEnvelope = { - /** - * The version of specification format for this particular entity that - * this is written against. - */ - apiVersion: string; - - /** - * The high level entity type being described. - */ - kind: string; - - /** - * Optional metadata related to the entity. - */ - metadata?: EntityMeta; - - /** - * The specification data describing the entity itself. - */ - spec?: object; -}; - -/** - * Parses and validates descriptors. - * - * The output must be validated and well formed. - */ -export type DescriptorParser = { - /** - * Parses and validates a single raw descriptor. - * - * @param descriptor A raw descriptor object - * @returns A structure describing the parsed and validated descriptor - * @throws An Error if the descriptor was malformed - */ - parse(descriptor: object): Promise; -}; - -/** - * Parses and validates a single envelope into its materialized kind. - * - * These parsers may assume that the envelope is already validated and well - * formed. - */ -export type KindParser = { - /** - * Try to parse an envelope into a materialized kind. - * - * @param envelope A valid descriptor envelope - * @returns A materialized type, or undefined if the given version/kind is - * not meant to be handled by this parser - * @throws An Error if the type was handled and found to not be properly - * formatted - */ - tryParse( - envelope: DescriptorEnvelope, - ): Promise; -}; - -export class ParserError extends Error { - constructor(message?: string, private _entityName?: string | undefined) { - super(message); - } - get entityName() { - return this._entityName; - } -} - -export type ReaderOutput = - | { type: 'error'; error: Error } - | { type: 'data'; data: object }; - -export type LocationReader = { - /** - * Reads the contents of a single location. - * - * @param type The type of location to read - * @param target The location target (type-specific) - * @returns The parsed contents, as an array of unverified descriptors or - * errors where the individual documents could not be parsed. - * @throws An error if the location as a whole could not be read - */ - read(type: string, target: string): Promise; -}; - -export type LocationSource = { - /** - * Reads the contents of a single location. - * - * @param target The location target to read - * @returns The parsed contents, as an array of unverified descriptors - * @throws An error if the location target could not be read - */ - read(target: string): Promise; +export type IngestionModel = { + readLocation(type: string, target: string): Promise; }; diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 8c29362015..e405e296b1 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -15,10 +15,10 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; -import { DescriptorEnvelope } from '../ingestion'; import { createRouter } from './router'; class MockEntitiesCatalog implements EntitiesCatalog { @@ -37,7 +37,7 @@ class MockLocationsCatalog implements LocationsCatalog { describe('createRouter', () => { describe('entities', () => { it('happy path: lists entities', async () => { - const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }]; + const entities: Entity[] = [{ apiVersion: 'a', kind: 'b' }]; const catalog = new MockEntitiesCatalog(); catalog.entities.mockResolvedValueOnce(entities); @@ -76,7 +76,7 @@ describe('createRouter', () => { describe('entityByUid', () => { it('can fetch entity by uid', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { @@ -117,7 +117,7 @@ describe('createRouter', () => { describe('entityByName', () => { it('can fetch entity by name', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index c3318a7377..0678cd3aef 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -22,7 +22,7 @@ import { addLocationSchema, EntitiesCatalog, EntityFilters, - LocationsCatalog, + LocationsCatalog } from '../catalog'; import { validateRequestBody } from './util'; diff --git a/yarn.lock b/yarn.lock index 6390797c12..74b9e451b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17169,11 +17169,6 @@ rc@^1.0.1, rc@^1.1.6, rc@^1.2.7, rc@^1.2.8: minimist "^1.2.0" strip-json-comments "~2.0.1" -react-addons-text-content@0.0.4: - version "0.0.4" - resolved "https://registry.npmjs.org/react-addons-text-content/-/react-addons-text-content-0.0.4.tgz#d2e259fdc951d1d8906c08902002108dce8792e5" - integrity sha1-0uJZ/clR0diQbAiQIAIQjc6HkuU= - react-beautiful-dnd@11.0.3: version "11.0.3" resolved "https://registry.npmjs.org/react-beautiful-dnd/-/react-beautiful-dnd-11.0.3.tgz#5678bb3e725d8b56cb7cf57f56e952105fc4f2af"