From cdd1d89a43d479839af44f82c6fd4010fee6b654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 15:46:55 +0200 Subject: [PATCH 1/7] Rename DescriptorEnvelope to Entity --- .../src/catalog/DatabaseEntitiesCatalog.ts | 8 +++---- .../catalog/DatabaseLocationsCatalog.test.ts | 5 ++--- .../src/catalog/DatabaseLocationsCatalog.ts | 2 +- .../src/catalog/StaticEntitiesCatalog.ts | 12 +++++----- plugins/catalog-backend/src/catalog/types.ts | 8 +++---- .../src/database/Database.test.ts | 10 ++++----- .../catalog-backend/src/database/Database.ts | 22 ++++++++++--------- .../src/database/DatabaseManager.ts | 9 +++----- .../src/database/search.test.ts | 6 ++--- .../catalog-backend/src/database/search.ts | 4 ++-- plugins/catalog-backend/src/database/types.ts | 6 ++--- .../src/ingestion/DescriptorParsers.ts | 9 ++------ .../ComponentDescriptorV1beta1Parser.ts | 8 +++---- .../descriptors/DescriptorEnvelopeParser.ts | 8 +++---- .../catalog-backend/src/ingestion/types.ts | 8 +++---- .../src/service/router.test.ts | 12 +++++----- 16 files changed, 64 insertions(+), 73 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 9d0d8fcaf7..d14b9df658 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -15,20 +15,20 @@ */ import { Database } from '../database'; -import { DescriptorEnvelope } from '../ingestion/types'; +import { Entity } from '../ingestion/types'; import { EntitiesCatalog, EntityFilters } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { constructor(private readonly database: Database) {} - async entities(filters?: EntityFilters): Promise { + async entities(filters?: EntityFilters): Promise { const items = await this.database.transaction(tx => this.database.entities(tx, filters), ); return items.map(i => i.entity); } - async entityByUid(uid: string): Promise { + async entityByUid(uid: string): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [{ key: 'uid', values: [uid] }]), ); @@ -40,7 +40,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [ { key: 'kind', values: [kind] }, diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 0181b7fc28..a63dd7d497 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +import { getVoidLogger } from '@backstage/backend-common'; import knex from 'knex'; import path from 'path'; - import { Database } from '../database'; import { ReaderOutput } from '../ingestion/types'; -import { getVoidLogger } from '@backstage/backend-common'; +import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; describe('DatabaseLocationsCatalog', () => { const database = knex({ diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 6e3c9ae306..d82d8e2c26 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -15,8 +15,8 @@ */ import { Database } from '../database'; -import { AddLocation, Location, LocationsCatalog } from './types'; import { LocationReader } from '../ingestion'; +import { AddLocation, Location, LocationsCatalog } from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 371cdf1f76..64cee69dd7 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -16,21 +16,21 @@ import { NotFoundError } from '@backstage/backend-common'; import lodash from 'lodash'; -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; import { EntitiesCatalog } from './types'; export class StaticEntitiesCatalog implements EntitiesCatalog { - private _entities: DescriptorEnvelope[]; + private _entities: Entity[]; - constructor(entities: DescriptorEnvelope[]) { + constructor(entities: Entity[]) { this._entities = entities; } - async entities(): Promise { + async entities(): Promise { return lodash.cloneDeep(this._entities); } - async entityByUid(uid: string): Promise { + async entityByUid(uid: string): Promise { const item = this._entities.find(e => uid === e.metadata?.uid); if (!item) { throw new NotFoundError('Entity cannot be found'); @@ -42,7 +42,7 @@ export class StaticEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const item = this._entities.find( e => kind === e.kind && diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index d5627ea86a..90bb6943b3 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -15,7 +15,7 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; // // Entities @@ -28,13 +28,13 @@ export type EntityFilter = { export type EntityFilters = EntityFilter[]; export type EntitiesCatalog = { - entities(filters?: EntityFilters): Promise; - entityByUid(uid: string): Promise; + entities(filters?: EntityFilters): Promise; + entityByUid(uid: string): Promise; entityByName( kind: string, namespace: string | undefined, name: string, - ): Promise; + ): Promise; }; // diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index a82385aae4..18481455ea 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -21,7 +21,7 @@ import { } from '@backstage/backend-common'; import Knex from 'knex'; import path from 'path'; -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; import { Database } from './Database'; import { AddDatabaseLocation, @@ -247,8 +247,8 @@ describe('Database', () => { describe('entities', () => { it('can get all entities with empty filters list', async () => { const catalog = new Database(database, getVoidLogger()); - const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' }; - const e2: DescriptorEnvelope = { + const e1: Entity = { apiVersion: 'a', kind: 'b' }; + const e2: Entity = { apiVersion: 'a', kind: 'b', spec: { c: null }, @@ -271,7 +271,7 @@ describe('Database', () => { it('can get all specific entities for matching filters (naive case)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ + const entities: Entity[] = [ { apiVersion: 'a', kind: 'b' }, { apiVersion: 'a', @@ -305,7 +305,7 @@ describe('Database', () => { it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ + const entities: Entity[] = [ { apiVersion: 'a', kind: 'b' }, { apiVersion: 'a', diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index 614f7d09bf..b7777d3ccb 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -24,7 +24,7 @@ import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { EntityFilters } from '../catalog'; -import { DescriptorEnvelope, EntityMeta } from '../ingestion'; +import { Entity, EntityMeta } from '../ingestion'; import { buildEntitySearch } from './search'; import { AddDatabaseLocation, @@ -54,9 +54,7 @@ function serializeMetadata(metadata: EntityMeta | undefined): string | null { return JSON.stringify(getStrippedMetadata(metadata)); } -function serializeSpec( - spec: DescriptorEnvelope['spec'], -): DbEntitiesRow['spec'] { +function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] { if (!spec) { return null; } @@ -66,7 +64,7 @@ function serializeSpec( function toEntityRow( locationId: string | undefined, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesRow { return { id: entity.metadata!.uid!, @@ -83,7 +81,7 @@ function toEntityRow( } function toEntityResponse(row: DbEntitiesRow): DbEntityResponse { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: row.api_version, kind: row.kind, metadata: { @@ -94,7 +92,7 @@ function toEntityResponse(row: DbEntitiesRow): DbEntityResponse { }; if (row.metadata) { - const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata']; + const metadata = JSON.parse(row.metadata) as Entity['metadata']; entity.metadata = { ...entity.metadata, ...metadata }; } @@ -127,7 +125,9 @@ function generateUid(): string { } function generateEtag(): string { - return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, ''); + return Buffer.from(uuidv4(), 'utf8') + .toString('base64') + .replace(/[^\w]/g, ''); } /** @@ -374,7 +374,9 @@ export class Database { target, }); - return (await tx('locations').where({ id }).select())![0]; + return (await tx('locations') + .where({ id }) + .select())![0]; }); } @@ -422,7 +424,7 @@ export class Database { private async updateEntitiesSearch( tx: Knex.Transaction, entityId: string, - data: DescriptorEnvelope, + data: Entity, ): Promise { try { const entries = buildEntitySearch(entityId, data); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 7f646971db..4b7fa38326 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -19,8 +19,8 @@ import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; import { - DescriptorEnvelope, DescriptorParser, + Entity, LocationReader, ParserError, } from '../ingestion'; @@ -129,7 +129,7 @@ export class DatabaseManager { private static async refreshSingleEntity( database: Database, locationId: string, - entity: DescriptorEnvelope, + entity: Entity, logger: Logger, ): Promise { const { kind } = entity; @@ -163,10 +163,7 @@ export class DatabaseManager { }); } - private static entitiesAreEqual( - first: DescriptorEnvelope, - second: DescriptorEnvelope, - ) { + private static entitiesAreEqual(first: Entity, second: Entity) { const firstClone = lodash.cloneDeep(first); const secondClone = lodash.cloneDeep(second); diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts index 26b20d1846..4c1df3429e 100644 --- a/plugins/catalog-backend/src/database/search.test.ts +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; import { buildEntitySearch, visitEntityPart } from './search'; import { DbEntitiesSearchRow } from './types'; @@ -99,7 +99,7 @@ describe('search', () => { describe('buildEntitySearch', () => { it('adds special keys even if missing', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', }; @@ -116,7 +116,7 @@ describe('search', () => { }); it('adds prefix-stripped versions', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', metadata: { diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts index f54cfde7e6..e14ebd338f 100644 --- a/plugins/catalog-backend/src/database/search.ts +++ b/plugins/catalog-backend/src/database/search.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; import { DbEntitiesSearchRow } from './types'; // Search entries that start with these prefixes, also get a shorthand without @@ -119,7 +119,7 @@ export function visitEntityPart( */ export function buildEntitySearch( entityId: string, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesSearchRow[] { // Start with some special keys that are always present because you want to // be able to easily search for null specifically diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 06e234545f..9fe4a523cc 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -15,7 +15,7 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; export type DbEntitiesRow = { id: string; @@ -32,12 +32,12 @@ export type DbEntitiesRow = { export type DbEntityRequest = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntityResponse = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntitiesSearchRow = { diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts index 86283df666..6375351ebd 100644 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts @@ -17,12 +17,7 @@ import { makeValidator } from '../validation'; import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; -import { - DescriptorEnvelope, - DescriptorParser, - KindParser, - ParserError, -} from './types'; +import { DescriptorParser, Entity, KindParser, ParserError } from './types'; export class DescriptorParsers implements DescriptorParser { static create(): DescriptorParser { @@ -37,7 +32,7 @@ export class DescriptorParsers implements DescriptorParser { private readonly kindParsers: KindParser[], ) {} - async parse(descriptor: object): Promise { + async parse(descriptor: object): Promise { const envelope = await this.envelopeParser.parse(descriptor); for (const parser of this.kindParsers) { const parsed = await parser.tryParse(envelope); diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts index 974b34aa8e..0934231a0c 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts @@ -15,9 +15,9 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope, KindParser, ParserError } from '../types'; +import { Entity, KindParser, ParserError } from '../types'; -export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope { +export interface ComponentDescriptorV1beta1 extends Entity { spec: { type: string; }; @@ -41,9 +41,7 @@ export class ComponentDescriptorV1beta1Parser implements KindParser { }); } - async tryParse( - envelope: DescriptorEnvelope, - ): Promise { + async tryParse(envelope: Entity): Promise { if ( envelope.apiVersion !== 'backstage.io/v1beta1' || envelope.kind !== 'Component' diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts index 1f21e2df9e..3833d22e77 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts +++ b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts @@ -16,13 +16,13 @@ import * as yup from 'yup'; import { Validators } from '../../validation'; -import { DescriptorEnvelope } from '../types'; +import { Entity } from '../types'; /** * Parses some raw structured data as a descriptor envelope */ export class DescriptorEnvelopeParser { - private schema: yup.Schema; + private schema: yup.Schema; constructor(validators: Validators) { const apiVersionSchema = yup @@ -160,8 +160,8 @@ export class DescriptorEnvelopeParser { .noUnknown(); } - async parse(data: any): Promise { - let result: DescriptorEnvelope; + async parse(data: any): Promise { + let result: Entity; try { result = await this.schema.validate(data, { strict: true }); } catch (e) { diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index e5083a65f1..45ee3bc638 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -87,7 +87,7 @@ export type EntityMeta = { * * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ */ -export type DescriptorEnvelope = { +export type Entity = { /** * The version of specification format for this particular entity that * this is written against. @@ -123,7 +123,7 @@ export type DescriptorParser = { * @returns A structure describing the parsed and validated descriptor * @throws An Error if the descriptor was malformed */ - parse(descriptor: object): Promise; + parse(descriptor: object): Promise; }; /** @@ -142,9 +142,7 @@ export type KindParser = { * @throws An Error if the type was handled and found to not be properly * formatted */ - tryParse( - envelope: DescriptorEnvelope, - ): Promise; + tryParse(envelope: Entity): Promise; }; export class ParserError extends Error { diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 8c29362015..210ad98577 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -18,7 +18,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import express from 'express'; import request from 'supertest'; import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '../ingestion'; import { createRouter } from './router'; class MockEntitiesCatalog implements EntitiesCatalog { @@ -37,7 +37,7 @@ class MockLocationsCatalog implements LocationsCatalog { describe('createRouter', () => { describe('entities', () => { it('happy path: lists entities', async () => { - const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }]; + const entities: Entity[] = [{ apiVersion: 'a', kind: 'b' }]; const catalog = new MockEntitiesCatalog(); catalog.entities.mockResolvedValueOnce(entities); @@ -76,7 +76,7 @@ describe('createRouter', () => { describe('entityByUid', () => { it('can fetch entity by uid', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { @@ -117,7 +117,7 @@ describe('createRouter', () => { describe('entityByName', () => { it('can fetch entity by name', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { @@ -190,7 +190,9 @@ describe('createRouter', () => { }); const app = express().use(router); - const response = await request(app).post('/locations').send(location); + const response = await request(app) + .post('/locations') + .send(location); expect(response.status).toEqual(400); }); From add55f64a86c41081901431765e7447eaa2af55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 11:52:50 +0200 Subject: [PATCH 2/7] Create the packages/catalog-model package --- packages/catalog-model/.eslintrc.js | 3 + packages/catalog-model/README.md | 12 + packages/catalog-model/package.json | 34 +++ packages/catalog-model/src/EntityPolicies.ts | 88 ++++++++ packages/catalog-model/src/entity/Entity.ts | 108 +++++++++ packages/catalog-model/src/entity/index.ts | 18 ++ .../policies/FieldFormatEntityPolicy.test.ts | 105 +++++++++ .../policies/FieldFormatEntityPolicy.ts | 91 ++++++++ .../ForeignRootFieldsEntityPolicy.test.ts | 52 +++++ .../policies/ForeignRootFieldsEntityPolicy.ts | 40 ++++ .../ReservedFieldsEntityPolicy.test.ts | 62 ++++++ .../policies/ReservedFieldsEntityPolicy.ts | 66 ++++++ .../policies/SchemaValidEntityPolicy.test.ts | 176 +++++++++++++++ .../policies/SchemaValidEntityPolicy.ts | 80 +++++++ .../src/entity/policies/index.ts | 20 ++ packages/catalog-model/src/index.ts | 21 ++ .../src/kinds/ComponentV1beta1.ts | 63 ++++++ packages/catalog-model/src/kinds/index.ts | 20 ++ packages/catalog-model/src/setupTests.ts | 15 ++ packages/catalog-model/src/types.ts | 32 +++ .../CommonValidatorFunctions.test.ts | 178 +++++++++++++++ .../validation/CommonValidatorFunctions.ts | 108 +++++++++ .../KubernetesValidatorFunctions.test.ts | 209 ++++++++++++++++++ .../KubernetesValidatorFunctions.ts | 86 +++++++ .../catalog-model/src/validation/index.ts | 20 ++ .../src/validation/makeValidator.ts | 38 ++++ .../catalog-model/src/validation/types.ts | 27 +++ ...0200520140700_location_update_log_table.ts | 5 +- .../src/ingestion/IngestionModels.ts | 73 ++++++ .../ingestion/descriptor/DescriptorParsers.ts | 45 ++++ .../src/ingestion/descriptor/index.ts | 18 ++ .../parsers/YamlDescriptorParser.ts | 64 ++++++ .../src/ingestion/descriptor/parsers/types.ts | 42 ++++ .../src/ingestion/source/LocationReaders.ts | 41 ++++ .../src/ingestion/source/index.ts | 20 ++ .../source/readers/FileLocationReader.ts | 35 +++ .../readers/GitHubLocationReader.test.ts | 92 ++++++++ .../source/readers/GitHubLocationReader.ts | 74 +++++++ .../src/ingestion/source/readers/types.ts | 29 +++ yarn.lock | 5 - 40 files changed, 2309 insertions(+), 6 deletions(-) create mode 100644 packages/catalog-model/.eslintrc.js create mode 100644 packages/catalog-model/README.md create mode 100644 packages/catalog-model/package.json create mode 100644 packages/catalog-model/src/EntityPolicies.ts create mode 100644 packages/catalog-model/src/entity/Entity.ts create mode 100644 packages/catalog-model/src/entity/index.ts create mode 100644 packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts create mode 100644 packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts create mode 100644 packages/catalog-model/src/entity/policies/index.ts create mode 100644 packages/catalog-model/src/index.ts create mode 100644 packages/catalog-model/src/kinds/ComponentV1beta1.ts create mode 100644 packages/catalog-model/src/kinds/index.ts create mode 100644 packages/catalog-model/src/setupTests.ts create mode 100644 packages/catalog-model/src/types.ts create mode 100644 packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts create mode 100644 packages/catalog-model/src/validation/CommonValidatorFunctions.ts create mode 100644 packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts create mode 100644 packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts create mode 100644 packages/catalog-model/src/validation/index.ts create mode 100644 packages/catalog-model/src/validation/makeValidator.ts create mode 100644 packages/catalog-model/src/validation/types.ts create mode 100644 plugins/catalog-backend/src/ingestion/IngestionModels.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts create mode 100644 plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/LocationReaders.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts create mode 100644 plugins/catalog-backend/src/ingestion/source/readers/types.ts diff --git a/packages/catalog-model/.eslintrc.js b/packages/catalog-model/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/packages/catalog-model/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/packages/catalog-model/README.md b/packages/catalog-model/README.md new file mode 100644 index 0000000000..755b9ee63c --- /dev/null +++ b/packages/catalog-model/README.md @@ -0,0 +1,12 @@ +# Catalog Model + +Contains the core model types and validators/policies used by the Backstage catalog functionality. + +This package will be imported both by the frontend and backend parts of the catalog, +as well as by others that want to consume catalog data. + +## Links + +- (Default frontend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog] +- (Default backend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog-backend] +- (The Backstage homepage)[https://backstage.io] diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json new file mode 100644 index 0000000000..ffd8fdba22 --- /dev/null +++ b/packages/catalog-model/package.json @@ -0,0 +1,34 @@ +{ + "name": "@backstage/catalog-model", + "version": "0.1.1-alpha.6", + "main": "dist/index.esm.js", + "main:src": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "lodash": "^4.17.15", + "yup": "^0.28.5" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.6", + "@types/jest": "^25.2.2", + "@types/lodash": "^4.14.151", + "@types/yup": "^0.28.2", + "yaml": "^1.9.2" + }, + "files": [ + "dist/**/*.{js,d.ts}" + ] +} diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts new file mode 100644 index 0000000000..cedf0df13d --- /dev/null +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + FieldFormatEntityPolicy, + ForeignRootFieldsEntityPolicy, + ReservedFieldsEntityPolicy, + SchemaValidEntityPolicy, +} from './entity'; +import { ComponentV1beta1Policy } from './kinds'; +import { EntityPolicy } from './types'; + +// Helper that requires that all of a set of policies can be successfully +// applied +class AllEntityPolicies implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + let result = entity; + for (const policy of this.policies) { + result = await policy.apply(entity); + } + return result; + } +} + +// Helper that requires that at least one of a set of policies can be +// successfully applied +class AnyEntityPolicy implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + for (const policy of this.policies) { + try { + return await policy.apply(entity); + } catch { + continue; + } + } + throw new Error(`The entity did not match any known policy`); + } +} + +export class EntityPolicies implements EntityPolicy { + private readonly policy: EntityPolicy; + + static defaultPolicies(): EntityPolicy { + return EntityPolicies.allOf([ + EntityPolicies.allOf([ + new SchemaValidEntityPolicy(), + new ForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy(), + new ReservedFieldsEntityPolicy(), + ]), + EntityPolicies.anyOf([new ComponentV1beta1Policy()]), + ]); + } + + static allOf(policies: EntityPolicy[]): EntityPolicy { + return new AllEntityPolicies(policies); + } + + static anyOf(policies: EntityPolicy[]): EntityPolicy { + return new AnyEntityPolicy(policies); + } + + constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) { + this.policy = policy; + } + + apply(entity: Entity): Promise { + return this.policy.apply(entity); + } +} diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts new file mode 100644 index 0000000000..6f57626749 --- /dev/null +++ b/packages/catalog-model/src/entity/Entity.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The format envelope that's common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type Entity = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Optional metadata related to the entity. + */ + metadata?: EntityMeta; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Metadata fields common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type EntityMeta = { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace, for any given kind. + */ + name?: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: Record; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: Record; +}; diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts new file mode 100644 index 0000000000..9e96021336 --- /dev/null +++ b/packages/catalog-model/src/entity/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { Entity, EntityMeta } from './Entity'; +export * from './policies'; diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts new file mode 100644 index 0000000000..d81b1155be --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; + +describe('FieldFormatEntityPolicy', () => { + let data: any; + let policy: FieldFormatEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new FieldFormatEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad apiVersion', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + data.apiVersion = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad kind', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + data.kind = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('handles missing metadata gracefully', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('handles missing spec gracefully', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name.*7/); + data.metadata.name = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/name.*aaaa/); + }); + + it('rejects bad namespace', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace.*7/); + data.metadata.namespace = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/namespace.*aaaa/); + }); + + it('rejects bad label key', async () => { + data.metadata.labels['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad label value', async () => { + data.metadata.labels.a = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad annotation key', async () => { + data.metadata.annotations['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*a#b/i); + }); + + it('rejects bad annotation value', async () => { + data.metadata.annotations.a = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*7/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts new file mode 100644 index 0000000000..1f94354f3f --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { makeValidator, Validators } from '../../validation'; +import { Entity } from '../Entity'; + +/** + * Ensures that the format of individual fields of the entity envelope + * is valid. + * + * This does not take into account machine generated fields such as uid, etag + * and generation. + */ +export class FieldFormatEntityPolicy implements EntityPolicy { + private readonly validators: Validators; + + constructor(validators: Validators = makeValidator()) { + this.validators = validators; + } + + async apply(entity: Entity): Promise { + function require( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + if (value === undefined || value === null) { + throw new Error(`${field} must have a value`); + } + + let isValid: boolean; + try { + isValid = validator(value); + } catch (e) { + throw new Error(`${field} could not be validated, ${e}`); + } + + if (!isValid) { + throw new Error(`${field} "${value}" is not valid`); + } + } + + function optional( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + return value === undefined || require(field, value, validator); + } + + require('apiVersion', entity.apiVersion, this.validators.isValidApiVersion); + require('kind', entity.kind, this.validators.isValidKind); + + optional( + 'metadata.name', + entity.metadata?.name, + this.validators.isValidEntityName, + ); + optional( + 'metadata.namespace', + entity.metadata?.namespace, + this.validators.isValidNamespace, + ); + + for (const [k, v] of Object.entries(entity.metadata?.labels ?? [])) { + require(`labels.${k}`, k, this.validators.isValidLabelKey); + require(`labels.${k}`, v, this.validators.isValidLabelValue); + } + + for (const [k, v] of Object.entries(entity.metadata?.annotations ?? [])) { + require(`annotations.${k}`, k, this.validators.isValidAnnotationKey); + require(`annotations.${k}`, v, this.validators.isValidAnnotationValue); + } + + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..98299259f8 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; + +describe('ForeignRootFieldsEntityPolicy', () => { + let data: any; + let policy: ForeignRootFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ForeignRootFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects unknown root fields', async () => { + data.spec2 = {}; + await expect(policy.apply(data)).rejects.toThrow(/spec2/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts new file mode 100644 index 0000000000..a4733e9a42 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec']; + +/** + * Ensures that there are no foreign root fields in the entity. + */ +export class ForeignRootFieldsEntityPolicy implements EntityPolicy { + private readonly knownFields: string[]; + + constructor(knownFields: string[] = defaultKnownFields) { + this.knownFields = knownFields; + } + + async apply(entity: Entity): Promise { + for (const field of Object.keys(entity)) { + if (!this.knownFields.includes(field)) { + throw new Error(`Unknown field ${field}`); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..348eabbdac --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; + +describe('ReservedFieldsEntityPolicy', () => { + let data: any; + let policy: ReservedFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ReservedFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects reserved keys in the spec root', async () => { + data.spec.apiVersion = 'a/b'; + await expect(policy.apply(data)).rejects.toThrow(/spec.*apiVersion/i); + }); + + it('rejects reserved keys in labels', async () => { + data.metadata.labels.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/label.*apiVersion/i); + }); + + it('rejects reserved keys in annotations', async () => { + data.metadata.annotations.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*apiVersion/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts new file mode 100644 index 0000000000..be2f732ca4 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_RESERVED_ENTITY_FIELDS = [ + 'apiVersion', + 'kind', + 'uid', + 'etag', + 'generation', + 'name', + 'namespace', + 'labels', + 'annotations', + 'spec', +]; + +/** + * Ensures that fields are not given certain reserved names. + */ +export class ReservedFieldsEntityPolicy implements EntityPolicy { + private readonly reservedFields: string[]; + + constructor(fields?: string[]) { + this.reservedFields = [ + ...(fields ?? []), + ...DEFAULT_RESERVED_ENTITY_FIELDS, + ]; + } + + async apply(entity: Entity): Promise { + for (const field of this.reservedFields) { + if (entity.spec?.hasOwnProperty(field)) { + throw new Error( + `The spec may not contain the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.labels?.hasOwnProperty(field)) { + throw new Error( + `A label may not have the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.annotations?.hasOwnProperty(field)) { + throw new Error( + `An annotation may not have the field ${field}, because it has reserved meaning`, + ); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts new file mode 100644 index 0000000000..b9d1165a60 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { Entity } from '../Entity'; +import { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; + +describe('SchemaValidEntityPolicy', () => { + let data: any; + let policy: SchemaValidEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new SchemaValidEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + // + // apiVersion and kind + // + + it('rejects wrong root type', async () => { + await expect(policy.apply((7 as unknown) as Entity)).rejects.toThrow( + /object/, + ); + }); + + it('rejects missing apiVersion', async () => { + delete data.apiVersion; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad apiVersion type', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects missing kind', async () => { + delete data.kind; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('rejects bad kind type', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + // + // metadata + // + + it('accepts missing metadata', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad metadata type', async () => { + data.metadata = 7; + await expect(policy.apply(data)).rejects.toThrow(/metadata/); + }); + + it('accepts missing uid', async () => { + delete data.metadata.uid; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad uid type', async () => { + data.metadata.uid = 7; + await expect(policy.apply(data)).rejects.toThrow(/uid/); + }); + + it('accepts missing etag', async () => { + delete data.metadata.etag; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad etag type', async () => { + data.metadata.etag = 7; + await expect(policy.apply(data)).rejects.toThrow(/etag/); + }); + + it('accepts missing generation', async () => { + delete data.metadata.generation; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad generation type', async () => { + data.metadata.generation = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/generation/); + }); + + it('accepts missing name', async () => { + delete data.metadata.name; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name type', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name/); + }); + + it('accepts missing namespace', async () => { + delete data.metadata.namespace; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad namespace type', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace/); + }); + + it('accepts missing labels', async () => { + delete data.metadata.labels; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad labels type', async () => { + data.metadata.labels = 7; + await expect(policy.apply(data)).rejects.toThrow(/labels/); + }); + + it('accepts missing annotations', async () => { + delete data.metadata.annotations; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad annotations type', async () => { + data.metadata.annotations = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotations/); + }); + + // + // spec + // + + it('accepts missing spec', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects non-object spec', async () => { + data.spec = 7; + await expect(policy.apply(data)).rejects.toThrow(/spec/); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts new file mode 100644 index 0000000000..7c0f5c20b6 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_ENTITY_SCHEMA = yup.object({ + apiVersion: yup.string().required(), + kind: yup.string().required(), + metadata: yup + .object({ + uid: yup + .string() + .notRequired() + .test( + 'metadata.uid', + 'The uid must not be empty', + value => value === undefined || value.length > 0, + ), + etag: yup + .string() + .notRequired() + .test( + 'metadata.etag', + 'The etag must not be empty', + value => value === undefined || value.length > 0, + ), + generation: yup + .number() + .notRequired() + .test( + 'metadata.generation', + 'The generation must be an integer greater than zero', + value => value === undefined || (value === (value | 0) && value > 0), + ), + name: yup.string().notRequired(), + namespace: yup.string().notRequired(), + labels: yup.object>().notRequired(), + annotations: yup.object>().notRequired(), + }) + .notRequired(), + spec: yup.object({}).notRequired(), +}); + +/** + * Ensures that the entity spec is valid according to a schema. + * + * This should be the first policy in the list, to ensure that other downstream + * policies can work with a structure that is at least valid in therms of the + * typescript type. + */ +export class SchemaValidEntityPolicy implements EntityPolicy { + private readonly schema: yup.Schema; + + constructor(schema: yup.Schema = DEFAULT_ENTITY_SCHEMA) { + this.schema = schema; + } + + async apply(entity: Entity): Promise { + try { + return await this.schema.validate(entity, { strict: true }); + } catch (e) { + throw new Error(`Malformed envelope, ${e}`); + } + } +} diff --git a/packages/catalog-model/src/entity/policies/index.ts b/packages/catalog-model/src/entity/policies/index.ts new file mode 100644 index 0000000000..f43aa68049 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; +export { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; +export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; +export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts new file mode 100644 index 0000000000..fb51461053 --- /dev/null +++ b/packages/catalog-model/src/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './entity'; +export { EntityPolicies } from './EntityPolicies'; +export * from './kinds'; +export type { EntityPolicy } from './types'; +export * from './validation'; diff --git a/packages/catalog-model/src/kinds/ComponentV1beta1.ts b/packages/catalog-model/src/kinds/ComponentV1beta1.ts new file mode 100644 index 0000000000..b0a3f627a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/ComponentV1beta1.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import type { Entity, EntityMeta } from '../entity/Entity'; +import type { EntityPolicy } from '../types'; + +const API_VERSION = 'backstage.io/v1beta1'; +const KIND = 'Component'; + +export interface ComponentV1beta1 extends Entity { + apiVersion: typeof API_VERSION; + kind: typeof KIND; + metadata: EntityMeta & { + name: string; + }; + spec: { + type: string; + }; +} + +export class ComponentV1beta1Policy implements EntityPolicy { + private schema: yup.Schema; + + constructor() { + this.schema = yup.object>({ + metadata: yup + .object({ + name: yup.string().required(), + }) + .required(), + spec: yup + .object({ + type: yup.string().required(), + }) + .required(), + }); + } + + async apply(envelope: Entity): Promise { + if ( + envelope.apiVersion !== 'backstage.io/v1beta1' || + envelope.kind !== 'Component' + ) { + throw new Error('Unsupported apiVersion / kind'); + } + + return await this.schema.validate(envelope, { strict: true }); + } +} diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts new file mode 100644 index 0000000000..97d22c14a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ComponentV1beta1 } from './ComponentV1beta1'; +export { ComponentV1beta1Policy } from './ComponentV1beta1'; +export { ComponentV1beta1 as Component }; +export { ComponentV1beta1 }; diff --git a/packages/catalog-model/src/setupTests.ts b/packages/catalog-model/src/setupTests.ts new file mode 100644 index 0000000000..f3b69cc361 --- /dev/null +++ b/packages/catalog-model/src/setupTests.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts new file mode 100644 index 0000000000..1d581cf23f --- /dev/null +++ b/packages/catalog-model/src/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from './entity/Entity'; + +/** + * A policy for validation or mutation to be applied to entities as they are + * entering the system. + */ +export type EntityPolicy = { + /** + * Applies validation or mutation on an entity. + * + * @param entity The entity, as validated/mutated so far in the policy tree + * @returns The incoming entity, or a mutated version of the same + * @throws An error if the entity should be rejected + */ + apply(entity: Entity): Promise; +}; diff --git a/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts new file mode 100644 index 0000000000..200e90b406 --- /dev/null +++ b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; + +describe('CommonValidatorFunctions', () => { + describe('isValidPrefixAndOrSuffix', () => { + it('only accepts strings', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + null, + '/', + () => true, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 7, + '/', + () => true, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + () => 'hello', + '/', + () => true, + () => true, + ), + ).toBe(false); + }); + + it('only accepts one or two parts', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b/c', + '/', + () => true, + () => true, + ), + ).toBe(false); + }); + + it('checks the prefix and suffix', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => false, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => false, + ), + ).toBe(false); + }); + }); + + it.each([ + [null, true], + [undefined, false], + [1, true], + ['a', true], + [() => 'a', false], + [Symbol('a'), false], + [[], true], + [[1], true], + [[undefined], false], + [{}, true], + [{ a: 1 }, true], + [{ a: undefined }, false], + ] as [any, boolean][])(`isJsonSafe %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isJsonSafe(value)).toBe(result); + }); + + it.each([ + [null, false], + [7, false], + ['', false], + ['a', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + ['adam.bertil.caesar', true], + ['adam.ber-til.caesar', true], + ['adam.-bertil.caesar', false], + ['adam.bertil-.caesar', false], + ['adam/bertil.caesar', false], + [`a.${'b'.repeat(63)}.c`, true], + [`a.${'b'.repeat(64)}.c`, false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(61)}`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(62)}`, + false, + ], + ])(`isValidDnsSubdomain %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isValidDnsSubdomain(value)).toBe(result); + }); + + it.each([ + [null, false], + [7, false], + ['', false], + ['a', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + [`${'a'.repeat(63)}`, true], + [`${'a'.repeat(64)}`, false], + ])(`isValidDnsLabel %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isValidDnsLabel(value)).toBe(result); + }); + + it.each([ + ['', ''], + ['a', 'a'], + ['a-b', 'ab'], + ['-a-b', 'ab'], + ['a_b', 'ab'], + [`${'a'.repeat(6000)}`, `${'a'.repeat(6000)}`], + ['_:;>!"#€', ''], + ])(`normalizeToLowercaseAlphanum %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.normalizeToLowercaseAlphanum(value)).toBe( + result, + ); + }); +}); diff --git a/packages/catalog-model/src/validation/CommonValidatorFunctions.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts new file mode 100644 index 0000000000..96a91aca06 --- /dev/null +++ b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import lodash from 'lodash'; + +/** + * Contains various helper validation and normalization functions that can be + * composed to form a Validator. + */ +export class CommonValidatorFunctions { + /** + * Checks that the value is on the form or , and validates + * those parts separately. + * + * @param value The value to check + * @param separator The separator between parts + * @param isValidPrefix Checks that the part before the separator is valid, if present + * @param isValidSuffix Checks that the part after the separator (or the entire value if there is no separator) is valid + */ + static isValidPrefixAndOrSuffix( + value: any, + separator: string, + isValidPrefix: (value: string) => boolean, + isValidSuffix: (value: string) => boolean, + ): boolean { + if (typeof value !== 'string') { + return false; + } + + const parts = value.split(separator); + if (parts.length === 1) { + return isValidSuffix(parts[0]); + } else if (parts.length === 2) { + return isValidPrefix(parts[0]) && isValidSuffix(parts[1]); + } + + return false; + } + + /** + * Checks that the value can be safely transferred as JSON. + * + * @param value The value to check + */ + static isJsonSafe(value: any): boolean { + try { + return lodash.isEqual(value, JSON.parse(JSON.stringify(value))); + } catch { + return false; + } + } + + /** + * Checks that the value is a valid DNS subdomain name. + * + * @param value The value to check + * @see https://tools.ietf.org/html/rfc1123 + */ + static isValidDnsSubdomain(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 253 && + value.split('.').every(CommonValidatorFunctions.isValidDnsLabel) + ); + } + + /** + * Checks that the value is a valid DNS label. + * + * @param value The value to check + * @see https://tools.ietf.org/html/rfc1123 + */ + static isValidDnsLabel(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-z0-9]+(\-[a-z0-9]+)*$/.test(value) + ); + } + + /** + * Normalizes by keeping only a-z, A-Z, and 0-9; and converts to lowercase. + * + * @param value The value to normalize + */ + static normalizeToLowercaseAlphanum(value: string): string { + return value + .split('') + .filter(x => /[a-zA-Z0-9]/.test(x)) + .join('') + .toLowerCase(); + } +} diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts new file mode 100644 index 0000000000..d0673085b4 --- /dev/null +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts @@ -0,0 +1,209 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; + +describe('KubernetesValidatorFunctions', () => { + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a-b', false], + ['a_b', false], + ['a.b', false], + ['a/a', true], + ['a/aAb5C', true], + ['a-b.c/v1', true], + ['a--b.c/v1', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/v1`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/v1`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidApiVersion %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidApiVersion(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['9AZ', false], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a-b', false], + ])(`isValidKind %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidKind(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ])(`isValidObjectName %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', false], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + ['a.b', false], + ])(`isValidNamespace %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidNamespace(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ['a/a', true], + ['a-b.c/a', true], + ['a--b.c/a', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/a`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/a`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidLabelKey %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidLabelKey(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', true], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ])(`isValidLabelValue %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ['a/a', true], + ['a-b.c/a', true], + ['a--b.c/a', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/a`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/a`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidAnnotationKey %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidAnnotationKey(value)).toBe( + matches, + ); + }); + + it.each([ + [7, false], + [null, false], + ['', true], + ['a', true], + ['/'.repeat(6000), true], + ])(`isValidAnnotationValue %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidAnnotationValue(value)).toBe( + matches, + ); + }); +}); diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts new file mode 100644 index 0000000000..fa938f5fcb --- /dev/null +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; + +/** + * Contains validation functions that match the Kubernetes spec, usable to + * build a catalog that is compatible with those rule sets. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + */ +export class KubernetesValidatorFunctions { + static isValidApiVersion(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + n => n.length >= 1 && n.length <= 63 && /^[a-z0-9A-Z]+$/.test(n), + ); + } + + static isValidKind(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-zA-Z][a-z0-9A-Z]*$/.test(value) + ); + } + + static isValidObjectName(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value) + ); + } + + static isValidNamespace(value: any): boolean { + return CommonValidatorFunctions.isValidDnsLabel(value); + } + + static isValidLabelKey(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + KubernetesValidatorFunctions.isValidObjectName, + ); + } + + static isValidLabelValue(value: any): boolean { + return ( + value === '' || KubernetesValidatorFunctions.isValidObjectName(value) + ); + } + + static isValidAnnotationKey(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + KubernetesValidatorFunctions.isValidObjectName, + ); + } + + static isValidAnnotationValue(value: any): boolean { + return typeof value === 'string'; + } +} diff --git a/packages/catalog-model/src/validation/index.ts b/packages/catalog-model/src/validation/index.ts new file mode 100644 index 0000000000..d679a5323c --- /dev/null +++ b/packages/catalog-model/src/validation/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CommonValidatorFunctions } from './CommonValidatorFunctions'; +export { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +export { makeValidator } from './makeValidator'; +export type { Validators } from './types'; diff --git a/packages/catalog-model/src/validation/makeValidator.ts b/packages/catalog-model/src/validation/makeValidator.ts new file mode 100644 index 0000000000..7ca01365e0 --- /dev/null +++ b/packages/catalog-model/src/validation/makeValidator.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; +import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +import { Validators } from './types'; + +const defaultValidators: Validators = { + isValidApiVersion: KubernetesValidatorFunctions.isValidApiVersion, + isValidKind: KubernetesValidatorFunctions.isValidKind, + isValidEntityName: KubernetesValidatorFunctions.isValidObjectName, + isValidNamespace: KubernetesValidatorFunctions.isValidNamespace, + normalizeEntityName: CommonValidatorFunctions.normalizeToLowercaseAlphanum, + isValidLabelKey: KubernetesValidatorFunctions.isValidLabelKey, + isValidLabelValue: KubernetesValidatorFunctions.isValidLabelValue, + isValidAnnotationKey: KubernetesValidatorFunctions.isValidAnnotationKey, + isValidAnnotationValue: KubernetesValidatorFunctions.isValidAnnotationValue, +}; + +export function makeValidator(overrides: Partial = {}): Validators { + return { + ...defaultValidators, + ...overrides, + }; +} diff --git a/packages/catalog-model/src/validation/types.ts b/packages/catalog-model/src/validation/types.ts new file mode 100644 index 0000000000..81209bfb75 --- /dev/null +++ b/packages/catalog-model/src/validation/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type Validators = { + isValidApiVersion(value: any): boolean; + isValidKind(value: any): boolean; + isValidEntityName(value: any): boolean; + isValidNamespace(value: any): boolean; + normalizeEntityName(value: string): string; + isValidLabelKey(value: any): boolean; + isValidLabelValue(value: any): boolean; + isValidAnnotationKey(value: any): boolean; + isValidAnnotationValue(value: any): boolean; +}; diff --git a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts index b2e1dc0d32..6700f5748e 100644 --- a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts +++ b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts @@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise { return knex.schema.createTable('location_update_log', table => { table.uuid('id').primary(); table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable(); table.string('message'); table .uuid('location_id') diff --git a/plugins/catalog-backend/src/ingestion/IngestionModels.ts b/plugins/catalog-backend/src/ingestion/IngestionModels.ts new file mode 100644 index 0000000000..616f1b49ae --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/IngestionModels.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model'; +import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types'; +import { LocationReader, LocationReaders } from './source'; +import { IngestionModel } from './types'; +import { DescriptorParsers } from './descriptor'; + +export class IngestionModels implements IngestionModel { + private readonly reader: LocationReader; + private readonly parser: DescriptorParser; + private readonly entityPolicy: EntityPolicy; + + static default(): IngestionModel { + return new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); + } + + constructor( + reader: LocationReader, + parser: DescriptorParser, + entityPolicy: EntityPolicy, + ) { + this.reader = reader; + this.parser = parser; + this.entityPolicy = entityPolicy; + } + + async readLocation(type: string, target: string): Promise { + const buffer = await this.reader.tryRead(type, target); + if (!buffer) { + throw new Error(`No reader could handle location ${type} ${target}`); + } + + const items = await this.parser.tryParse(buffer); + if (!items) { + throw new Error(`No parser could handle location ${type} ${target}`); + } + + const result: ReaderOutput[] = []; + for (const item of items) { + if (item.type === 'error') { + result.push(item); + } else { + try { + const output = await this.entityPolicy.apply(item.data); + result.push({ type: 'data', data: output }); + } catch (e) { + result.push({ type: 'error', error: e }); + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts new file mode 100644 index 0000000000..ed05855109 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DescriptorParser, ReaderOutput } from './parsers/types'; +import { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; + +/** + * Parses raw descriptor data (e.g. from a file or stream) into entities. + */ +export class DescriptorParsers implements DescriptorParser { + private readonly parsers: DescriptorParser[]; + + static defaultParsers(): DescriptorParser[] { + return [new YamlDescriptorParser()]; + } + + constructor( + parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(), + ) { + this.parsers = parsers; + } + + async tryParse(data: Buffer): Promise { + for (const parser of this.parsers) { + const result = await parser.tryParse(data); + if (result) { + return result; + } + } + throw new Error(`Unsupported descriptor format`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/index.ts b/plugins/catalog-backend/src/ingestion/descriptor/index.ts new file mode 100644 index 0000000000..1529c78afc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DescriptorParsers } from './DescriptorParsers'; +export { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts new file mode 100644 index 0000000000..3f2e9e3c5c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import yaml from 'yaml'; +import { DescriptorParser, ReaderOutput } from './types'; + +/** + * Parses descriptors on YAML format + */ +export class YamlDescriptorParser implements DescriptorParser { + async tryParse(data: Buffer): Promise { + // TODO(freben): Should perhaps first do format detection, so the parse + // failure can be emitted as a proper error instead of just as if we + // weren't handling the format at all. + let documents; + try { + documents = yaml.parseAllDocuments(data.toString('utf8')); + } catch (e) { + return undefined; + } + + const result: ReaderOutput[] = []; + + for (const document of documents) { + if (document.contents) { + if (document.errors?.length) { + result.push({ + type: 'error', + error: new Error(`Malformed YAML document, ${document.errors[0]}`), + }); + } else { + const json = document.toJSON(); + if (typeof json !== 'object' || Array.isArray(json)) { + result.push({ + type: 'error', + error: new Error(`Malformed descriptor, expected object at root`), + }); + } else { + result.push({ + type: 'data', + data: json as Entity, + }); + } + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts new file mode 100644 index 0000000000..baef5dc3df --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; + +export type ReaderOutput = + | { type: 'error'; error: Error } + | { type: 'data'; data: Entity }; + +/** + * Parses raw descriptor data (e.g. from a file) into entities. + */ +export type DescriptorParser = { + /** + * Try to parse some raw data into an entity. + * + * Note that this is only the low level operation of parsing the raw file + * format, e.g. reading JSON or YAML or similar and emitting as structured + * but unvalidated data. The actual validation is performed by EntityPolicy + * and KindParser. + * + * @param data Raw descriptor data + * @returns A list of raw unvalidated entities / errors, or undefined if the + * given data is not meant to be handled by this parser + * @throws An Error if the format was handled and found to not be properly + * formed + */ + tryParse(data: Buffer): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts new file mode 100644 index 0000000000..a670f309ad --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileLocationReader } from './readers/FileLocationReader'; +import { GitHubLocationReader } from './readers/GitHubLocationReader'; +import { LocationReader } from './readers/types'; + +export class LocationReaders implements LocationReader { + private readonly readers: LocationReader[]; + + static defaultReaders(): LocationReader[] { + return [new FileLocationReader(), new GitHubLocationReader()]; + } + + constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) { + this.readers = readers; + } + + async tryRead(type: string, target: string): Promise { + for (const reader of this.readers) { + const result = await reader.tryRead(type, target); + if (result) { + return result; + } + } + throw new Error(`Could not read unknown location "${type}", "${target}"`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/index.ts b/plugins/catalog-backend/src/ingestion/source/index.ts new file mode 100644 index 0000000000..3ed1063878 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { LocationReaders } from './LocationReaders'; +export { FileLocationReader } from './readers/FileLocationReader'; +export { GitHubLocationReader } from './readers/GitHubLocationReader'; +export { LocationReader } from './readers/types'; diff --git a/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts new file mode 100644 index 0000000000..0c64aebf14 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { LocationReader } from './types'; + +/** + * Reads a file from the local file system. + */ +export class FileLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'file') { + return undefined; + } + + try { + return await fs.readFile(target); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts new file mode 100644 index 0000000000..1f0f6ed539 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('node-fetch'); + +import fetch from 'node-fetch'; +import { GitHubLocationReader } from './GitHubLocationReader'; + +const { Response } = jest.requireActual('node-fetch'); + +describe('Unit: GitHubLocationReader', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('fetches the file and parses it correctly', async () => { + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + const reader = new GitHubLocationReader(); + const buffer = await reader.tryRead( + 'github', + 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', + ); + + expect(buffer?.toString('utf8')).toBe('hello'); + }); + + it('changes the url to point to https://raw.githubusercontent.com', async () => { + const gitHubUrl = `https://github.com`; + const project = `spotify/backstage`; + const folderPath = `master/plugins/catalog-backend/fixtures`; + const componentFilename = `one_component.yaml`; + const rawGitHubUrl = `https://raw.githubusercontent.com`; + + const reader = new GitHubLocationReader(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + await reader.tryRead( + 'github', + `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, + ); + + expect(fetch).toHaveBeenCalledWith( + `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`, + ); + }); + + describe('rejects wrong urls', () => { + const reader = new GitHubLocationReader(); + + it.each([ + ['http://example.com/one_component.yaml'], + ['http://github.com/one_component.yaml'], + ['http://github.com/PROJECT/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.json'], + ])( + '%p', + async (url: string) => + await expect(reader.tryRead('github', url)).rejects.toThrow(/url/), + ); + }); +}); + +describe('Integration: GitHubLocationSource', () => { + beforeAll(() => { + (fetch as any).mockImplementation(jest.requireActual('node-fetch')); + }); + + it('fetches the fixture from backstage repo', async () => { + const PERMANENT_LINK = + 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; + + const reader = new GitHubLocationReader(); + const result = await reader.tryRead('github', PERMANENT_LINK); + + expect(result?.toString('utf8')).toContain('component3'); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts new file mode 100644 index 0000000000..bd330e28b4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { URL } from 'url'; +import { LocationReader } from './types'; + +/** + * Reads a file whose target is a GitHub URL. + * + * Uses raw.githubusercontent.com for now, but this will probably change in the + * future when token auth is implemented. + */ +export class GitHubLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'github') { + return undefined; + } + + const url = this.buildRawUrl(target); + try { + return await fetch(url.toString()).then(x => x.buffer()); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } + + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/readers/types.ts b/plugins/catalog-backend/src/ingestion/source/readers/types.ts new file mode 100644 index 0000000000..37c7b46885 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type LocationReader = { + /** + * Reads the contents of a single location. + * + * @param type The type of location to read + * @param target The location target (type-specific) + * @returns The target contents, as a raw Buffer, or undefined if this type + * was not meant to be consumed by this reader + * @throws An error if the type was meant for this reader, but could not be + * read + */ + tryRead(type: string, target: string): Promise; +}; diff --git a/yarn.lock b/yarn.lock index 6390797c12..74b9e451b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17169,11 +17169,6 @@ rc@^1.0.1, rc@^1.1.6, rc@^1.2.7, rc@^1.2.8: minimist "^1.2.0" strip-json-comments "~2.0.1" -react-addons-text-content@0.0.4: - version "0.0.4" - resolved "https://registry.npmjs.org/react-addons-text-content/-/react-addons-text-content-0.0.4.tgz#d2e259fdc951d1d8906c08902002108dce8792e5" - integrity sha1-0uJZ/clR0diQbAiQIAIQjc6HkuU= - react-beautiful-dnd@11.0.3: version "11.0.3" resolved "https://registry.npmjs.org/react-beautiful-dnd/-/react-beautiful-dnd-11.0.3.tgz#5678bb3e725d8b56cb7cf57f56e952105fc4f2af" From 1d6324a92d8fc3a835b890cdb1e44e957b3730fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 11:52:50 +0200 Subject: [PATCH 3/7] Move the model into a plugin-catalog-model for sharing outside the backend --- packages/backend/package.json | 13 +- packages/backend/src/plugins/catalog.ts | 14 +- plugins/catalog-backend/package.json | 5 +- .../src/catalog/DatabaseEntitiesCatalog.ts | 2 +- .../catalog/DatabaseLocationsCatalog.test.ts | 35 +-- .../src/catalog/DatabaseLocationsCatalog.ts | 14 +- .../src/catalog/StaticEntitiesCatalog.ts | 2 +- plugins/catalog-backend/src/catalog/types.ts | 2 +- .../src/database/Database.test.ts | 2 +- .../catalog-backend/src/database/Database.ts | 2 +- .../src/database/DatabaseManager.test.ts | 82 ++++--- .../src/database/DatabaseManager.ts | 25 +-- .../src/database/search.test.ts | 2 +- .../catalog-backend/src/database/search.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 2 +- .../src/ingestion/DescriptorParsers.ts | 48 ---- .../src/ingestion/LocationReaders.ts | 39 ---- .../ComponentDescriptorV1beta1Parser.ts | 61 ----- .../DescriptorEnvelopeParser.test.ts | 172 -------------- .../descriptors/DescriptorEnvelopeParser.ts | 206 ----------------- .../catalog-backend/src/ingestion/index.ts | 7 +- .../ingestion/sources/FileLocationSource.ts | 36 --- .../ingestion/sources/GitHubLocationSource.ts | 73 ------ .../__tests__/GitHubLocationSource.test.ts | 110 --------- .../src/ingestion/sources/util.ts | 55 ----- .../catalog-backend/src/ingestion/types.ts | 168 +------------- .../src/service/router.test.ts | 2 +- plugins/catalog-backend/src/service/router.ts | 2 +- .../CommonValidatorFunctions.test.ts | 178 --------------- .../validation/CommonValidatorFunctions.ts | 108 --------- .../KubernetesValidatorFunctions.test.ts | 209 ------------------ .../KubernetesValidatorFunctions.ts | 86 ------- .../catalog-backend/src/validation/index.ts | 20 -- .../src/validation/makeValidator.ts | 38 ---- .../catalog-backend/src/validation/types.ts | 27 --- 35 files changed, 119 insertions(+), 1730 deletions(-) delete mode 100644 plugins/catalog-backend/src/ingestion/DescriptorParsers.ts delete mode 100644 plugins/catalog-backend/src/ingestion/LocationReaders.ts delete mode 100644 plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts delete mode 100644 plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts delete mode 100644 plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts delete mode 100644 plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts delete mode 100644 plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/sources/util.ts delete mode 100644 plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts delete mode 100644 plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts delete mode 100644 plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts delete mode 100644 plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts delete mode 100644 plugins/catalog-backend/src/validation/index.ts delete mode 100644 plugins/catalog-backend/src/validation/makeValidator.ts delete mode 100644 plugins/catalog-backend/src/validation/types.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 9aedafecbb..51fe04e51d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,7 +10,7 @@ }, "scripts": { "build": "tsc", - "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess nodemon", + "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"nodemon -r esm\\\"", "lint": "backstage-cli lint", "test": "backstage-cli test", "clean": "backstage-cli clean", @@ -18,13 +18,15 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@backstage/catalog-model": "^0.1.1-alpha.6", "@backstage/plugin-auth-backend": "^0.1.1-alpha.6", "@backstage/plugin-catalog-backend": "^0.1.1-alpha.6", - "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", - "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", "@backstage/plugin-identity-backend": "^0.1.1-alpha.6", + "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", + "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", + "esm": "^3.2.25", "express": "^4.17.1", "helmet": "^3.22.0", "knex": "^0.21.1", @@ -43,6 +45,9 @@ "typescript": "^3.9.2" }, "nodemonConfig": { - "watch": "./dist" + "watch": [ + "./dist", + "node_modules/@backstage*" + ] } } diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 687fd9157a..7e843cc80b 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -21,22 +21,28 @@ import { DatabaseManager, DescriptorParsers, LocationReaders, + IngestionModels, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; +import { EntityPolicies } from '@backstage/catalog-model'; export default async function ({ logger, database }: PluginEnvironment) { - const reader = LocationReaders.create(); - const parser = DescriptorParsers.create(); + const policy = new EntityPolicies(); + const ingestion = new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); const db = await DatabaseManager.createDatabase(database, logger); runPeriodically( - () => DatabaseManager.refreshLocations(db, reader, parser, logger), + () => DatabaseManager.refreshLocations(db, ingestion, policy, logger), 10000, ); const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db, reader); + const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion); return await createRouter({ entitiesCatalog, locationsCatalog, logger }); } diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f32ffffb6c..5ec37295ed 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,8 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", - "@types/node-fetch": "^2.5.7", - "@types/supertest": "^2.0.8", + "@backstage/catalog-model": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -38,6 +37,8 @@ "devDependencies": { "@backstage/cli": "^0.1.1-alpha.6", "@types/lodash": "^4.14.151", + "@types/node-fetch": "^2.5.7", + "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index d14b9df658..972410a639 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { Database } from '../database'; -import { Entity } from '../ingestion/types'; import { EntitiesCatalog, EntityFilters } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index a63dd7d497..56a3b3828f 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -14,12 +14,27 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import knex from 'knex'; import path from 'path'; import { Database } from '../database'; -import { ReaderOutput } from '../ingestion/types'; +import { IngestionModel } from '../ingestion/types'; import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +class MockIngestionModel implements IngestionModel { + readLocation = jest.fn(async (type: string, target: string) => { + if (type !== 'valid_type') { + throw new Error(`Unknown location type ${type}`); + } + if (target === 'valid_target') { + return [{ type: 'data', data: {} as Entity } as const]; + } + throw new Error( + `Can't read location at ${target} with error: Something is broken`, + ); + }); +} + describe('DatabaseLocationsCatalog', () => { const database = knex({ client: 'sqlite3', @@ -31,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => { }); let db: Database; let catalog: DatabaseLocationsCatalog; - - const mockLocationReader = { - read: async (type: string, target: string): Promise => { - if (type !== 'valid_type') { - throw new Error(`Unknown location type ${type}`); - } - if (target === 'valid_target') { - return Promise.resolve([{ type: 'data', data: {} }]); - } - throw new Error( - `Can't read location at ${target} with error: Something is broken`, - ); - }, - }; + let ingestionModel: IngestionModel; beforeEach(async () => { await database.migrate.latest({ @@ -52,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => { loadExtensions: ['.ts'], }); db = new Database(database, getVoidLogger()); - catalog = new DatabaseLocationsCatalog(db, mockLocationReader); + ingestionModel = new MockIngestionModel(); + catalog = new DatabaseLocationsCatalog(db, ingestionModel); }); it('resolves to location with id', async () => { diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index d82d8e2c26..b13d70de86 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -15,17 +15,25 @@ */ import { Database } from '../database'; -import { LocationReader } from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { AddLocation, Location, LocationsCatalog } from './types'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( private readonly database: Database, - private readonly reader: LocationReader, + private readonly ingestionModel: IngestionModel, ) {} async addLocation(location: AddLocation): Promise { - const outputs = await this.reader.read(location.type, location.target); + const outputs = await this.ingestionModel.readLocation( + location.type, + location.target, + ); + if (!outputs) { + throw new Error( + `Unknown location type ${location.type} ${location.target}`, + ); + } outputs.forEach(output => { if (output.type === 'error') { throw new Error( diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 64cee69dd7..1de606d44d 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -15,8 +15,8 @@ */ import { NotFoundError } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import lodash from 'lodash'; -import { Entity } from '../ingestion'; import { EntitiesCatalog } from './types'; export class StaticEntitiesCatalog implements EntitiesCatalog { diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 90bb6943b3..2f8967cd1b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { Entity } from '../ingestion'; // // Entities diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index 18481455ea..7a38adf02f 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -19,9 +19,9 @@ import { getVoidLogger, NotFoundError, } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import Knex from 'knex'; import path from 'path'; -import { Entity } from '../ingestion'; import { Database } from './Database'; import { AddDatabaseLocation, diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index b7777d3ccb..1f5cc7562f 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -19,12 +19,12 @@ import { InputError, NotFoundError, } from '@backstage/backend-common'; +import { Entity, EntityMeta } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { EntityFilters } from '../catalog'; -import { Entity, EntityMeta } from '../ingestion'; import { buildEntitySearch } from './search'; import { AddDatabaseLocation, diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index ca50f518cb..c8d0aec332 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -15,13 +15,9 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; -import { - ComponentDescriptor, - DescriptorParser, - LocationReader, - ParserError, -} from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; @@ -32,18 +28,18 @@ describe('DatabaseManager', () => { const db = ({ locations: jest.fn().mockResolvedValue([]), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(), + const reader: IngestionModel = { + readLocation: jest.fn(), }; - const parser: DescriptorParser = { - parse: jest.fn(), + const policy: EntityPolicy = { + apply: jest.fn(), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).not.toHaveBeenCalled(); - expect(parser.parse).not.toHaveBeenCalled(); + expect(reader.readLocation).not.toHaveBeenCalled(); + expect(policy.apply).not.toHaveBeenCalled(); }); it('can update a single location', async () => { @@ -52,7 +48,7 @@ describe('DatabaseManager', () => { type: 'some', target: 'thing', }; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, @@ -68,18 +64,20 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as Partial) as Database; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).toHaveBeenCalledTimes(1); - expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing'); + expect(reader.readLocation).toHaveBeenCalledTimes(1); + expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing'); expect(db.addEntity).toHaveBeenCalledTimes(1); expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, { locationId: '123', @@ -108,21 +106,23 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -158,23 +158,23 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), - }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), ), }; + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.reject(new Error('parser error message'))), + }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -211,19 +211,17 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(() => + const reader: IngestionModel = { + readLocation: jest.fn(() => Promise.reject([{ type: 'error', error: new Error('test message') }]), ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), - ), + const policy: EntityPolicy = { + apply: jest.fn(() => Promise.reject(new Error('parser error message'))), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 4b7fa38326..a3aded18ec 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,16 +14,12 @@ * limitations under the License. */ +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; -import { - DescriptorParser, - Entity, - LocationReader, - ParserError, -} from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { Database } from './Database'; import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types'; @@ -67,8 +63,8 @@ export class DatabaseManager { public static async refreshLocations( database: Database, - reader: LocationReader, - parser: DescriptorParser, + ingestionModel: IngestionModel, + entityPolicy: EntityPolicy, logger: Logger, ): Promise { const locations = await database.locations(); @@ -78,7 +74,10 @@ export class DatabaseManager { `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`, ); - const readerOutput = await reader.read(location.type, location.target); + const readerOutput = await ingestionModel.readLocation( + location.type, + location.target, + ); for (const readerItem of readerOutput) { if (readerItem.type === 'error') { @@ -87,7 +86,7 @@ export class DatabaseManager { } try { - const entity = await parser.parse(readerItem.data); + const entity = await entityPolicy.apply(readerItem.data); await DatabaseManager.refreshSingleEntity( database, location.id, @@ -100,15 +99,11 @@ export class DatabaseManager { entity.metadata!.name, ); } catch (error) { - let entityName; - if (error instanceof ParserError) { - entityName = error.entityName; - } await DatabaseManager.logUpdateFailure( database, location.id, error, - entityName, + readerItem.data.metadata?.name, ); } } diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts index 4c1df3429e..7ad06aee14 100644 --- a/plugins/catalog-backend/src/database/search.test.ts +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { buildEntitySearch, visitEntityPart } from './search'; import { DbEntitiesSearchRow } from './types'; diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts index e14ebd338f..fcacf1a9f2 100644 --- a/plugins/catalog-backend/src/database/search.ts +++ b/plugins/catalog-backend/src/database/search.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { DbEntitiesSearchRow } from './types'; // Search entries that start with these prefixes, also get a shorthand without diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 9fe4a523cc..ca1cbbdfd4 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { Entity } from '../ingestion'; export type DbEntitiesRow = { id: string; diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts deleted file mode 100644 index 6375351ebd..0000000000 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { makeValidator } from '../validation'; -import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; -import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; -import { DescriptorParser, Entity, KindParser, ParserError } from './types'; - -export class DescriptorParsers implements DescriptorParser { - static create(): DescriptorParser { - const validators = makeValidator(); - return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [ - new ComponentDescriptorV1beta1Parser(), - ]); - } - - constructor( - private readonly envelopeParser: DescriptorEnvelopeParser, - private readonly kindParsers: KindParser[], - ) {} - - async parse(descriptor: object): Promise { - const envelope = await this.envelopeParser.parse(descriptor); - for (const parser of this.kindParsers) { - const parsed = await parser.tryParse(envelope); - if (parsed) { - return parsed; - } - } - throw new ParserError( - `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`, - envelope.metadata?.name, - ); - } -} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts deleted file mode 100644 index aa88d299e4..0000000000 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { FileLocationSource } from './sources/FileLocationSource'; -import { GitHubLocationSource } from './sources/GitHubLocationSource'; -import { LocationReader, LocationSource, ReaderOutput } from './types'; - -export class LocationReaders implements LocationReader { - static create(): LocationReader { - return new LocationReaders({ - file: new FileLocationSource(), - github: new GitHubLocationSource(), - }); - } - - constructor(private readonly sources: Record) {} - - async read(type: string, target: string): Promise { - const source = this.sources[type]; - if (!source) { - throw new Error(`Unknown location type ${type}`); - } - - return source.read(target); - } -} diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts deleted file mode 100644 index 0934231a0c..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as yup from 'yup'; -import { Entity, KindParser, ParserError } from '../types'; - -export interface ComponentDescriptorV1beta1 extends Entity { - spec: { - type: string; - }; -} - -export class ComponentDescriptorV1beta1Parser implements KindParser { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - metadata: yup - .object({ - name: yup.string().required(), - }) - .required(), - spec: yup - .object({ - type: yup.string().required(), - }) - .required(), - }); - } - - async tryParse(envelope: Entity): Promise { - if ( - envelope.apiVersion !== 'backstage.io/v1beta1' || - envelope.kind !== 'Component' - ) { - return undefined; - } - - try { - return await this.schema.validate(envelope, { strict: true }); - } catch (e) { - throw new ParserError( - `Malformed component, ${e}`, - envelope.metadata?.name, - ); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts deleted file mode 100644 index 7c96fb7cf5..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { makeValidator } from '../../validation'; -import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser'; - -describe('DescriptorEnvelopeParser', () => { - let data: any; - let parser: DescriptorEnvelopeParser; - - beforeEach(() => { - data = yaml.parse(` - apiVersion: backstage.io/v1beta1 - kind: Component - metadata: - uid: e01199ab-08cc-44c2-8e19-5c29ded82521 - etag: lsndfkjsndfkjnsdfkjnsd== - generation: 13 - name: my-component-yay - namespace: the-namespace - labels: - backstage.io/custom: ValueStuff - annotations: - example.com/bindings: are-secret - spec: - custom: stuff - `); - parser = new DescriptorEnvelopeParser(makeValidator()); - }); - - it('works for the happy path', async () => { - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects missing apiVersion', async () => { - delete data.apiVersion; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects wrong root type', async () => { - await expect(parser.parse(7)).rejects.toThrow(/object/); - }); - - it('rejects bad apiVersion', async () => { - data.apiVersion = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects missing kind', async () => { - delete data.kind; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('rejects bad kind', async () => { - data.kind = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('accepts missing metadata', async () => { - delete data.medatata; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object metadata', async () => { - data.metadata = 7; - await expect(parser.parse(data)).rejects.toThrow(/metadata/); - }); - - it('accepts missing uid', async () => { - delete data.metadata.uid; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad uid', async () => { - data.metadata.uid = 7; - await expect(parser.parse(data)).rejects.toThrow(/uid/); - }); - - it('accepts missing etag', async () => { - delete data.metadata.etag; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad etag', async () => { - data.metadata.etag = 7; - await expect(parser.parse(data)).rejects.toThrow(/etag/); - }); - - it('accepts missing generation', async () => { - delete data.metadata.generation; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad generation', async () => { - data.metadata.generation = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/generation/); - }); - - it('accepts missing spec', async () => { - delete data.spec; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object spec', async () => { - data.spec = 7; - await expect(parser.parse(data)).rejects.toThrow(/spec/); - }); - - it('rejects bad name', async () => { - data.metadata.name = 7; - await expect(parser.parse(data)).rejects.toThrow(/name/); - }); - - it('rejects bad namespace', async () => { - data.metadata.namespace = 7; - await expect(parser.parse(data)).rejects.toThrow(/namespace/); - }); - - it('rejects bad label key', async () => { - data.metadata.labels['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/label.*key/i); - }); - - it('rejects bad label value', async () => { - data.metadata.labels.a = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/label.*value/i); - }); - - it('rejects bad annotation key', async () => { - data.metadata.annotations['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i); - }); - - it('rejects bad annotation value', async () => { - data.metadata.annotations.a = []; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i); - }); - - it('rejects unknown root keys', async () => { - data.spec2 = {}; - await expect(parser.parse(data)).rejects.toThrow(/spec2/i); - }); - - it('rejects reserved keys in the spec root', async () => { - data.spec.apiVersion = 'a/b'; - await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i); - }); - - it('rejects reserved keys in labels', async () => { - data.metadata.labels.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i); - }); - - it('rejects reserved keys in annotations', async () => { - data.metadata.annotations.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts deleted file mode 100644 index 3833d22e77..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as yup from 'yup'; -import { Validators } from '../../validation'; -import { Entity } from '../types'; - -/** - * Parses some raw structured data as a descriptor envelope - */ -export class DescriptorEnvelopeParser { - private schema: yup.Schema; - - constructor(validators: Validators) { - const apiVersionSchema = yup - .string() - .required() - .test( - 'apiVersion', - 'The apiVersion is not formatted according to schema', - validators.isValidApiVersion, - ); - - const kindSchema = yup - .string() - .required() - .test( - 'kind', - 'The kind is not formatted according to schema', - validators.isValidKind, - ); - - const uidSchema = yup - .string() - .notRequired() - .test( - 'metadata.uid', - 'The uid is not formatted according to schema', - value => value === undefined || value.length > 0, - ); - - const etagSchema = yup - .string() - .notRequired() - .test( - 'metadata.etag', - 'The etag value is not according to schema', - value => value === undefined || value.length > 0, - ); - - const generationSchema = yup - .number() - .notRequired() - .test( - 'metadata.generation', - 'The generation value is not according to schema', - value => value === undefined || value > 0, - ); - - const nameSchema = yup - .string() - .notRequired() - .test( - 'metadata.name', - 'The name is not formatted according to schema', - value => value === undefined || validators.isValidEntityName(value), - ); - - const namespaceSchema = yup - .string() - .notRequired() - .test( - 'metadata.namespace', - 'The namespace is malformed', - value => value === undefined || validators.isValidNamespace(value), - ); - - const labelsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.labels.keys', - message: 'Label keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidLabelKey) - ); - }, - }) - .test({ - name: 'metadata.labels.values', - message: 'Label values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidLabelValue) - ); - }, - }); - - const annotationsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.annotations.keys', - message: 'Annotation keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidAnnotationKey) - ); - }, - }) - .test({ - name: 'metadata.annotations.values', - message: 'Annotation values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidAnnotationValue) - ); - }, - }); - - const metadataSchema = yup - .object({ - uid: uidSchema, - etag: etagSchema, - generation: generationSchema, - name: nameSchema, - namespace: namespaceSchema, - labels: labelsSchema, - annotations: annotationsSchema, - }) - .notRequired(); - - const specSchema = yup.object({}).notRequired(); - - this.schema = yup - .object({ - apiVersion: apiVersionSchema, - kind: kindSchema, - metadata: metadataSchema, - spec: specSchema, - }) - .noUnknown(); - } - - async parse(data: any): Promise { - let result: Entity; - try { - result = await this.schema.validate(data, { strict: true }); - } catch (e) { - throw new Error(`Malformed envelope, ${e}`); - } - - // These are keys with specific semantic meaning in a document, that we do - // not want to appear in the root of the spec, or as labels or as - // annotations, because they will lead to confusion. - const reservedKeys = [ - 'apiVersion', - 'kind', - 'uid', - 'etag', - 'generation', - 'name', - 'namespace', - 'labels', - 'annotations', - 'spec', - ]; - for (const key of reservedKeys) { - if (result.spec?.hasOwnProperty(key)) { - throw new Error( - `The spec may not contain the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.labels?.hasOwnProperty(key)) { - throw new Error( - `A label may not have the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.annotations?.hasOwnProperty(key)) { - throw new Error( - `An annotation may not have the key ${key}, because it has reserved meaning`, - ); - } - } - - return result; - } -} diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index ca6f2dd2be..b6aceaecdd 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ -export * from './DescriptorParsers'; -export * from './LocationReaders'; -export * from './types'; +export * from './descriptor'; +export { IngestionModels } from './IngestionModels'; +export * from './source'; +export type { IngestionModel } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts deleted file mode 100644 index 9d2794ec4f..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; - -export class FileLocationSource implements LocationSource { - async read(target: string): Promise { - let rawYaml; - try { - rawYaml = await fs.readFile(target, 'utf8'); - } catch (e) { - throw new Error(`Unable to read "${target}", ${e}`); - } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts deleted file mode 100644 index 69ba4911a6..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fetch from 'node-fetch'; -import { URL } from 'url'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; - -// Pointing to raw.githubusercontent.com for now -// to be changed in the future, after auth and tokens are done -export class GitHubLocationSource implements LocationSource { - async read(target: string): Promise { - let url: URL; - - try { - url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; - url.protocol = 'https'; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - - let rawYaml; - try { - rawYaml = await fetch(url.toString()).then(x => { - return x.text(); - }); - } catch (e) { - throw new Error(`Unable to read "${target}", ${e}`); - } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } - } -} diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts deleted file mode 100644 index 083c2f7e86..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -jest.mock('node-fetch'); - -import fs from 'fs-extra'; -import fetch from 'node-fetch'; -import path from 'path'; -import { GitHubLocationSource } from '../GitHubLocationSource'; - -const { Response } = jest.requireActual('node-fetch'); - -const FIXTURES_DIR = path.resolve( - __dirname, - '..', - '..', - '..', - '..', - 'fixtures', -); -const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => { - acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8'); - return acc; -}, {} as Record); - -describe('Unit: GitHubLocationSource', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('fetches the file and parses it correctly', async () => { - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures['one_component.yaml'])), - ); - const reader = new GitHubLocationSource(); - - const result = await reader.read( - 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', - ); - - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); - }); - - it('changes the url to point to https://raw.githubusercontent.com', async () => { - const gitHubUrl = `https://github.com`; - const project = `spotify/backstage`; - const folderPath = `master/plugins/catalog-backend/fixtures`; - const componentFilename = `one_component.yaml`; - const rawGitHubUrl = `https://raw.githubusercontent.com`; - const reader = new GitHubLocationSource(); - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures[componentFilename])), - ); - - await reader.read( - `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, - ); - - expect(fetch).toHaveBeenCalledWith( - `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`, - ); - }); - - describe('rejects wrong urls', () => { - const reader = new GitHubLocationSource(); - - it.each([ - ['http://example.com/one_component.yaml'], - ['http://github.com/one_component.yaml'], - ['http://github.com/PROJECT/one_component.yaml'], - ['http://github.com/PROJECT/REPO/one_component.yaml'], - ['http://github.com/PROJECT/REPO/one_component.json'], - ])( - '%p', - async (url: string) => - await expect(reader.read(url)).rejects.toThrow(/url/), - ); - }); -}); - -describe('Integration: GitHubLocationSource', () => { - beforeAll(() => { - (fetch as any).mockImplementation(jest.requireActual('node-fetch')); - }); - - it('fetches the fixture from backstage repo', async () => { - const PERMANENT_LINK = - 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; - const reader = new GitHubLocationSource(); - - const result = await reader.read(PERMANENT_LINK); - - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/sources/util.ts b/plugins/catalog-backend/src/ingestion/sources/util.ts deleted file mode 100644 index cccbeb92b3..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/util.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { ReaderOutput } from '../types'; - -export function readDescriptorYaml(data: string): ReaderOutput[] { - let documents; - try { - documents = yaml.parseAllDocuments(data); - } catch (e) { - throw new Error(`Could not parse YAML data, ${e}`); - } - - const result: ReaderOutput[] = []; - - for (const document of documents) { - if (document.contents) { - if (document.errors?.length) { - result.push({ - type: 'error', - error: new Error(`Malformed YAML document, ${document.errors[0]}`), - }); - } else { - const json = document.toJSON(); - if (typeof json !== 'object' || Array.isArray(json)) { - result.push({ - type: 'error', - error: new Error(`Malformed descriptor, expected object at root`), - }); - } else { - result.push({ - type: 'data', - data: json, - }); - } - } - } - } - - return result; -} diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 45ee3bc638..8878c2af5b 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,170 +14,8 @@ * limitations under the License. */ -import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser'; +import { ReaderOutput } from './descriptor/parsers/types'; -export type ComponentDescriptor = ComponentDescriptorV1beta1; - -/** - * Metadata fields common to all versions/kinds of entity. - * - * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - */ -export type EntityMeta = { - /** - * A globally unique ID for the entity. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, but the server is free to reject requests - * that do so in such a way that it breaks semantics. - */ - uid?: string; - - /** - * An opaque string that changes for each update operation to any part of - * the entity, including metadata. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, and the server will then reject the - * operation if it does not match the current stored value. - */ - etag?: string; - - /** - * A positive nonzero number that indicates the current generation of data - * for this entity; the value is incremented each time the spec changes. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. - */ - generation?: number; - - /** - * The name of the entity. - * - * Must be uniqe within the catalog at any given point in time, for any - * given namespace, for any given kind. - */ - name?: string; - - /** - * The namespace that the entity belongs to. - */ - namespace?: string; - - /** - * Key/value pairs of identifying information attached to the entity. - */ - labels?: Record; - - /** - * Key/value pairs of non-identifying auxiliary information attached to the - * entity. - */ - annotations?: Record; -}; - -/** - * The format envelope that's common to all versions/kinds. - * - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ - */ -export type Entity = { - /** - * The version of specification format for this particular entity that - * this is written against. - */ - apiVersion: string; - - /** - * The high level entity type being described. - */ - kind: string; - - /** - * Optional metadata related to the entity. - */ - metadata?: EntityMeta; - - /** - * The specification data describing the entity itself. - */ - spec?: object; -}; - -/** - * Parses and validates descriptors. - * - * The output must be validated and well formed. - */ -export type DescriptorParser = { - /** - * Parses and validates a single raw descriptor. - * - * @param descriptor A raw descriptor object - * @returns A structure describing the parsed and validated descriptor - * @throws An Error if the descriptor was malformed - */ - parse(descriptor: object): Promise; -}; - -/** - * Parses and validates a single envelope into its materialized kind. - * - * These parsers may assume that the envelope is already validated and well - * formed. - */ -export type KindParser = { - /** - * Try to parse an envelope into a materialized kind. - * - * @param envelope A valid descriptor envelope - * @returns A materialized type, or undefined if the given version/kind is - * not meant to be handled by this parser - * @throws An Error if the type was handled and found to not be properly - * formatted - */ - tryParse(envelope: Entity): Promise; -}; - -export class ParserError extends Error { - constructor(message?: string, private _entityName?: string | undefined) { - super(message); - } - get entityName() { - return this._entityName; - } -} - -export type ReaderOutput = - | { type: 'error'; error: Error } - | { type: 'data'; data: object }; - -export type LocationReader = { - /** - * Reads the contents of a single location. - * - * @param type The type of location to read - * @param target The location target (type-specific) - * @returns The parsed contents, as an array of unverified descriptors or - * errors where the individual documents could not be parsed. - * @throws An error if the location as a whole could not be read - */ - read(type: string, target: string): Promise; -}; - -export type LocationSource = { - /** - * Reads the contents of a single location. - * - * @param target The location target to read - * @returns The parsed contents, as an array of unverified descriptors - * @throws An error if the location target could not be read - */ - read(target: string): Promise; +export type IngestionModel = { + readLocation(type: string, target: string): Promise; }; diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 210ad98577..36e8b55770 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -15,10 +15,10 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; -import { Entity } from '../ingestion'; import { createRouter } from './router'; class MockEntitiesCatalog implements EntitiesCatalog { diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index c3318a7377..0678cd3aef 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -22,7 +22,7 @@ import { addLocationSchema, EntitiesCatalog, EntityFilters, - LocationsCatalog, + LocationsCatalog } from '../catalog'; import { validateRequestBody } from './util'; diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts b/plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts deleted file mode 100644 index 200e90b406..0000000000 --- a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { CommonValidatorFunctions } from './CommonValidatorFunctions'; - -describe('CommonValidatorFunctions', () => { - describe('isValidPrefixAndOrSuffix', () => { - it('only accepts strings', () => { - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - null, - '/', - () => true, - () => true, - ), - ).toBe(false); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 7, - '/', - () => true, - () => true, - ), - ).toBe(false); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - () => 'hello', - '/', - () => true, - () => true, - ), - ).toBe(false); - }); - - it('only accepts one or two parts', () => { - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a', - '/', - () => true, - () => true, - ), - ).toBe(true); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a/b', - '/', - () => true, - () => true, - ), - ).toBe(true); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a/b/c', - '/', - () => true, - () => true, - ), - ).toBe(false); - }); - - it('checks the prefix and suffix', () => { - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a/b', - '/', - () => true, - () => true, - ), - ).toBe(true); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a/b', - '/', - () => false, - () => true, - ), - ).toBe(false); - expect( - CommonValidatorFunctions.isValidPrefixAndOrSuffix( - 'a/b', - '/', - () => true, - () => false, - ), - ).toBe(false); - }); - }); - - it.each([ - [null, true], - [undefined, false], - [1, true], - ['a', true], - [() => 'a', false], - [Symbol('a'), false], - [[], true], - [[1], true], - [[undefined], false], - [{}, true], - [{ a: 1 }, true], - [{ a: undefined }, false], - ] as [any, boolean][])(`isJsonSafe %p ? %p`, (value, result) => { - expect(CommonValidatorFunctions.isJsonSafe(value)).toBe(result); - }); - - it.each([ - [null, false], - [7, false], - ['', false], - ['a', true], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', false], - ['adam.bertil.caesar', true], - ['adam.ber-til.caesar', true], - ['adam.-bertil.caesar', false], - ['adam.bertil-.caesar', false], - ['adam/bertil.caesar', false], - [`a.${'b'.repeat(63)}.c`, true], - [`a.${'b'.repeat(64)}.c`, false], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(61)}`, - true, - ], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(62)}`, - false, - ], - ])(`isValidDnsSubdomain %p ? %p`, (value, result) => { - expect(CommonValidatorFunctions.isValidDnsSubdomain(value)).toBe(result); - }); - - it.each([ - [null, false], - [7, false], - ['', false], - ['a', true], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', false], - [`${'a'.repeat(63)}`, true], - [`${'a'.repeat(64)}`, false], - ])(`isValidDnsLabel %p ? %p`, (value, result) => { - expect(CommonValidatorFunctions.isValidDnsLabel(value)).toBe(result); - }); - - it.each([ - ['', ''], - ['a', 'a'], - ['a-b', 'ab'], - ['-a-b', 'ab'], - ['a_b', 'ab'], - [`${'a'.repeat(6000)}`, `${'a'.repeat(6000)}`], - ['_:;>!"#€', ''], - ])(`normalizeToLowercaseAlphanum %p ? %p`, (value, result) => { - expect(CommonValidatorFunctions.normalizeToLowercaseAlphanum(value)).toBe( - result, - ); - }); -}); diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts b/plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts deleted file mode 100644 index 96a91aca06..0000000000 --- a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import lodash from 'lodash'; - -/** - * Contains various helper validation and normalization functions that can be - * composed to form a Validator. - */ -export class CommonValidatorFunctions { - /** - * Checks that the value is on the form or , and validates - * those parts separately. - * - * @param value The value to check - * @param separator The separator between parts - * @param isValidPrefix Checks that the part before the separator is valid, if present - * @param isValidSuffix Checks that the part after the separator (or the entire value if there is no separator) is valid - */ - static isValidPrefixAndOrSuffix( - value: any, - separator: string, - isValidPrefix: (value: string) => boolean, - isValidSuffix: (value: string) => boolean, - ): boolean { - if (typeof value !== 'string') { - return false; - } - - const parts = value.split(separator); - if (parts.length === 1) { - return isValidSuffix(parts[0]); - } else if (parts.length === 2) { - return isValidPrefix(parts[0]) && isValidSuffix(parts[1]); - } - - return false; - } - - /** - * Checks that the value can be safely transferred as JSON. - * - * @param value The value to check - */ - static isJsonSafe(value: any): boolean { - try { - return lodash.isEqual(value, JSON.parse(JSON.stringify(value))); - } catch { - return false; - } - } - - /** - * Checks that the value is a valid DNS subdomain name. - * - * @param value The value to check - * @see https://tools.ietf.org/html/rfc1123 - */ - static isValidDnsSubdomain(value: any): boolean { - return ( - typeof value === 'string' && - value.length >= 1 && - value.length <= 253 && - value.split('.').every(CommonValidatorFunctions.isValidDnsLabel) - ); - } - - /** - * Checks that the value is a valid DNS label. - * - * @param value The value to check - * @see https://tools.ietf.org/html/rfc1123 - */ - static isValidDnsLabel(value: any): boolean { - return ( - typeof value === 'string' && - value.length >= 1 && - value.length <= 63 && - /^[a-z0-9]+(\-[a-z0-9]+)*$/.test(value) - ); - } - - /** - * Normalizes by keeping only a-z, A-Z, and 0-9; and converts to lowercase. - * - * @param value The value to normalize - */ - static normalizeToLowercaseAlphanum(value: string): string { - return value - .split('') - .filter(x => /[a-zA-Z0-9]/.test(x)) - .join('') - .toLowerCase(); - } -} diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts b/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts deleted file mode 100644 index d0673085b4..0000000000 --- a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; - -describe('KubernetesValidatorFunctions', () => { - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', true], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a-b', false], - ['a_b', false], - ['a.b', false], - ['a/a', true], - ['a/aAb5C', true], - ['a-b.c/v1', true], - ['a--b.c/v1', false], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 61, - )}/v1`, - true, - ], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 62, - )}/v1`, - false, - ], - [`a/${'a'.repeat(63)}`, true], - [`a/${'a'.repeat(64)}`, false], - ])(`isValidApiVersion %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidApiVersion(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', true], - ['9AZ', false], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a-b', false], - ])(`isValidKind %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidKind(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', true], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a/b', false], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', true], - ['a.b', true], - ])(`isValidObjectName %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', false], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a/b', false], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', false], - ['a.b', false], - ])(`isValidNamespace %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidNamespace(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', true], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a/b', true], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', true], - ['a.b', true], - ['a/a', true], - ['a-b.c/a', true], - ['a--b.c/a', false], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 61, - )}/a`, - true, - ], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 62, - )}/a`, - false, - ], - [`a/${'a'.repeat(63)}`, true], - [`a/${'a'.repeat(64)}`, false], - ])(`isValidLabelKey %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidLabelKey(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', true], - ['a', true], - ['AZ09', true], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a/b', false], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', true], - ['a.b', true], - ])(`isValidLabelValue %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches); - }); - - it.each([ - [7, false], - [null, false], - ['', false], - ['a', true], - ['AZ09', true], - ['a'.repeat(63), true], - ['a'.repeat(64), false], - ['a/b', true], - ['a-b', true], - ['-a-b', false], - ['a-b-', false], - ['a--b', false], - ['a_b', true], - ['a.b', true], - ['a/a', true], - ['a-b.c/a', true], - ['a--b.c/a', false], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 61, - )}/a`, - true, - ], - [ - `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( - 62, - )}/a`, - false, - ], - [`a/${'a'.repeat(63)}`, true], - [`a/${'a'.repeat(64)}`, false], - ])(`isValidAnnotationKey %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidAnnotationKey(value)).toBe( - matches, - ); - }); - - it.each([ - [7, false], - [null, false], - ['', true], - ['a', true], - ['/'.repeat(6000), true], - ])(`isValidAnnotationValue %p ? %p`, (value, matches) => { - expect(KubernetesValidatorFunctions.isValidAnnotationValue(value)).toBe( - matches, - ); - }); -}); diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts b/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts deleted file mode 100644 index fa938f5fcb..0000000000 --- a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { CommonValidatorFunctions } from './CommonValidatorFunctions'; - -/** - * Contains validation functions that match the Kubernetes spec, usable to - * build a catalog that is compatible with those rule sets. - * - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set - */ -export class KubernetesValidatorFunctions { - static isValidApiVersion(value: any): boolean { - return CommonValidatorFunctions.isValidPrefixAndOrSuffix( - value, - '/', - CommonValidatorFunctions.isValidDnsSubdomain, - n => n.length >= 1 && n.length <= 63 && /^[a-z0-9A-Z]+$/.test(n), - ); - } - - static isValidKind(value: any): boolean { - return ( - typeof value === 'string' && - value.length >= 1 && - value.length <= 63 && - /^[a-zA-Z][a-z0-9A-Z]*$/.test(value) - ); - } - - static isValidObjectName(value: any): boolean { - return ( - typeof value === 'string' && - value.length >= 1 && - value.length <= 63 && - /^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value) - ); - } - - static isValidNamespace(value: any): boolean { - return CommonValidatorFunctions.isValidDnsLabel(value); - } - - static isValidLabelKey(value: any): boolean { - return CommonValidatorFunctions.isValidPrefixAndOrSuffix( - value, - '/', - CommonValidatorFunctions.isValidDnsSubdomain, - KubernetesValidatorFunctions.isValidObjectName, - ); - } - - static isValidLabelValue(value: any): boolean { - return ( - value === '' || KubernetesValidatorFunctions.isValidObjectName(value) - ); - } - - static isValidAnnotationKey(value: any): boolean { - return CommonValidatorFunctions.isValidPrefixAndOrSuffix( - value, - '/', - CommonValidatorFunctions.isValidDnsSubdomain, - KubernetesValidatorFunctions.isValidObjectName, - ); - } - - static isValidAnnotationValue(value: any): boolean { - return typeof value === 'string'; - } -} diff --git a/plugins/catalog-backend/src/validation/index.ts b/plugins/catalog-backend/src/validation/index.ts deleted file mode 100644 index be607e43ec..0000000000 --- a/plugins/catalog-backend/src/validation/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export * from './CommonValidatorFunctions'; -export * from './KubernetesValidatorFunctions'; -export * from './makeValidator'; -export * from './types'; diff --git a/plugins/catalog-backend/src/validation/makeValidator.ts b/plugins/catalog-backend/src/validation/makeValidator.ts deleted file mode 100644 index 7ca01365e0..0000000000 --- a/plugins/catalog-backend/src/validation/makeValidator.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { CommonValidatorFunctions } from './CommonValidatorFunctions'; -import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; -import { Validators } from './types'; - -const defaultValidators: Validators = { - isValidApiVersion: KubernetesValidatorFunctions.isValidApiVersion, - isValidKind: KubernetesValidatorFunctions.isValidKind, - isValidEntityName: KubernetesValidatorFunctions.isValidObjectName, - isValidNamespace: KubernetesValidatorFunctions.isValidNamespace, - normalizeEntityName: CommonValidatorFunctions.normalizeToLowercaseAlphanum, - isValidLabelKey: KubernetesValidatorFunctions.isValidLabelKey, - isValidLabelValue: KubernetesValidatorFunctions.isValidLabelValue, - isValidAnnotationKey: KubernetesValidatorFunctions.isValidAnnotationKey, - isValidAnnotationValue: KubernetesValidatorFunctions.isValidAnnotationValue, -}; - -export function makeValidator(overrides: Partial = {}): Validators { - return { - ...defaultValidators, - ...overrides, - }; -} diff --git a/plugins/catalog-backend/src/validation/types.ts b/plugins/catalog-backend/src/validation/types.ts deleted file mode 100644 index 81209bfb75..0000000000 --- a/plugins/catalog-backend/src/validation/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type Validators = { - isValidApiVersion(value: any): boolean; - isValidKind(value: any): boolean; - isValidEntityName(value: any): boolean; - isValidNamespace(value: any): boolean; - normalizeEntityName(value: string): string; - isValidLabelKey(value: any): boolean; - isValidLabelValue(value: any): boolean; - isValidAnnotationKey(value: any): boolean; - isValidAnnotationValue(value: any): boolean; -}; From 0ce506ac43684de588c5dc823d4f40ebc1c8d097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 16:36:52 +0200 Subject: [PATCH 4/7] Less watching, less foreign root fields --- packages/backend/package.json | 5 +---- packages/catalog-model/src/EntityPolicies.ts | 4 ++-- ...cy.test.ts => NoForeignRootFieldsEntityPolicy.test.ts} | 8 ++++---- ...EntityPolicy.ts => NoForeignRootFieldsEntityPolicy.ts} | 2 +- packages/catalog-model/src/entity/policies/index.ts | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) rename packages/catalog-model/src/entity/policies/{ForeignRootFieldsEntityPolicy.test.ts => NoForeignRootFieldsEntityPolicy.test.ts} (85%) rename packages/catalog-model/src/entity/policies/{ForeignRootFieldsEntityPolicy.ts => NoForeignRootFieldsEntityPolicy.ts} (94%) diff --git a/packages/backend/package.json b/packages/backend/package.json index 51fe04e51d..e4b77c1e41 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -45,9 +45,6 @@ "typescript": "^3.9.2" }, "nodemonConfig": { - "watch": [ - "./dist", - "node_modules/@backstage*" - ] + "watch": "./dist" } } diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index cedf0df13d..abc953ab6f 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -17,7 +17,7 @@ import { Entity, FieldFormatEntityPolicy, - ForeignRootFieldsEntityPolicy, + NoForeignRootFieldsEntityPolicy, ReservedFieldsEntityPolicy, SchemaValidEntityPolicy, } from './entity'; @@ -62,7 +62,7 @@ export class EntityPolicies implements EntityPolicy { return EntityPolicies.allOf([ EntityPolicies.allOf([ new SchemaValidEntityPolicy(), - new ForeignRootFieldsEntityPolicy(), + new NoForeignRootFieldsEntityPolicy(), new FieldFormatEntityPolicy(), new ReservedFieldsEntityPolicy(), ]), diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts similarity index 85% rename from packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts rename to packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts index 98299259f8..190024bb29 100644 --- a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts @@ -15,11 +15,11 @@ */ import yaml from 'yaml'; -import { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; +import { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy'; -describe('ForeignRootFieldsEntityPolicy', () => { +describe('NoForeignRootFieldsEntityPolicy', () => { let data: any; - let policy: ForeignRootFieldsEntityPolicy; + let policy: NoForeignRootFieldsEntityPolicy; beforeEach(() => { data = yaml.parse(` @@ -38,7 +38,7 @@ describe('ForeignRootFieldsEntityPolicy', () => { spec: custom: stuff `); - policy = new ForeignRootFieldsEntityPolicy(); + policy = new NoForeignRootFieldsEntityPolicy(); }); it('works for the happy path', async () => { diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts similarity index 94% rename from packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts rename to packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts index a4733e9a42..3c0a2f5d61 100644 --- a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts @@ -22,7 +22,7 @@ const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec']; /** * Ensures that there are no foreign root fields in the entity. */ -export class ForeignRootFieldsEntityPolicy implements EntityPolicy { +export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { private readonly knownFields: string[]; constructor(knownFields: string[] = defaultKnownFields) { diff --git a/packages/catalog-model/src/entity/policies/index.ts b/packages/catalog-model/src/entity/policies/index.ts index f43aa68049..d64053f7cb 100644 --- a/packages/catalog-model/src/entity/policies/index.ts +++ b/packages/catalog-model/src/entity/policies/index.ts @@ -15,6 +15,6 @@ */ export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; -export { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; +export { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy'; export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; From 709e3381d8d6a1418a436200e9fd0445aebafd48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 20:50:46 +0200 Subject: [PATCH 5/7] Address comments --- packages/catalog-model/src/EntityPolicies.ts | 12 ++--- .../policies/FieldFormatEntityPolicy.test.ts | 30 +++++------ .../policies/FieldFormatEntityPolicy.ts | 2 +- .../NoForeignRootFieldsEntityPolicy.test.ts | 4 +- .../NoForeignRootFieldsEntityPolicy.ts | 2 +- .../ReservedFieldsEntityPolicy.test.ts | 10 ++-- .../policies/ReservedFieldsEntityPolicy.ts | 2 +- .../policies/SchemaValidEntityPolicy.test.ts | 52 +++++++++---------- .../policies/SchemaValidEntityPolicy.ts | 2 +- .../src/kinds/ComponentV1beta1.ts | 2 +- packages/catalog-model/src/setupTests.ts | 15 ------ packages/catalog-model/src/types.ts | 2 +- .../src/database/DatabaseManager.test.ts | 16 +++--- .../src/database/DatabaseManager.ts | 2 +- .../src/ingestion/IngestionModels.ts | 2 +- 15 files changed, 73 insertions(+), 82 deletions(-) delete mode 100644 packages/catalog-model/src/setupTests.ts diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index abc953ab6f..fa25334d5f 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -29,10 +29,10 @@ import { EntityPolicy } from './types'; class AllEntityPolicies implements EntityPolicy { constructor(private readonly policies: EntityPolicy[]) {} - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { let result = entity; for (const policy of this.policies) { - result = await policy.apply(entity); + result = await policy.enforce(entity); } return result; } @@ -43,10 +43,10 @@ class AllEntityPolicies implements EntityPolicy { class AnyEntityPolicy implements EntityPolicy { constructor(private readonly policies: EntityPolicy[]) {} - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { for (const policy of this.policies) { try { - return await policy.apply(entity); + return await policy.enforce(entity); } catch { continue; } @@ -82,7 +82,7 @@ export class EntityPolicies implements EntityPolicy { this.policy = policy; } - apply(entity: Entity): Promise { - return this.policy.apply(entity); + enforce(entity: Entity): Promise { + return this.policy.enforce(entity); } } diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts index d81b1155be..14b44108e5 100644 --- a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts @@ -42,64 +42,64 @@ describe('FieldFormatEntityPolicy', () => { }); it('works for the happy path', async () => { - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad apiVersion', async () => { data.apiVersion = 7; - await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); data.apiVersion = 'a#b'; - await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); }); it('rejects bad kind', async () => { data.kind = 7; - await expect(policy.apply(data)).rejects.toThrow(/kind/); + await expect(policy.enforce(data)).rejects.toThrow(/kind/); data.kind = 'a#b'; - await expect(policy.apply(data)).rejects.toThrow(/kind/); + await expect(policy.enforce(data)).rejects.toThrow(/kind/); }); it('handles missing metadata gracefully', async () => { delete data.medatata; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('handles missing spec gracefully', async () => { delete data.spec; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad name', async () => { data.metadata.name = 7; - await expect(policy.apply(data)).rejects.toThrow(/name.*7/); + await expect(policy.enforce(data)).rejects.toThrow(/name.*7/); data.metadata.name = 'a'.repeat(1000); - await expect(policy.apply(data)).rejects.toThrow(/name.*aaaa/); + await expect(policy.enforce(data)).rejects.toThrow(/name.*aaaa/); }); it('rejects bad namespace', async () => { data.metadata.namespace = 7; - await expect(policy.apply(data)).rejects.toThrow(/namespace.*7/); + await expect(policy.enforce(data)).rejects.toThrow(/namespace.*7/); data.metadata.namespace = 'a'.repeat(1000); - await expect(policy.apply(data)).rejects.toThrow(/namespace.*aaaa/); + await expect(policy.enforce(data)).rejects.toThrow(/namespace.*aaaa/); }); it('rejects bad label key', async () => { data.metadata.labels['a#b'] = 'value'; - await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i); }); it('rejects bad label value', async () => { data.metadata.labels.a = 'a#b'; - await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i); }); it('rejects bad annotation key', async () => { data.metadata.annotations['a#b'] = 'value'; - await expect(policy.apply(data)).rejects.toThrow(/annotation.*a#b/i); + await expect(policy.enforce(data)).rejects.toThrow(/annotation.*a#b/i); }); it('rejects bad annotation value', async () => { data.metadata.annotations.a = 7; - await expect(policy.apply(data)).rejects.toThrow(/annotation.*7/i); + await expect(policy.enforce(data)).rejects.toThrow(/annotation.*7/i); }); }); diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts index 1f94354f3f..697e45b371 100644 --- a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -32,7 +32,7 @@ export class FieldFormatEntityPolicy implements EntityPolicy { this.validators = validators; } - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { function require( field: string, value: any, diff --git a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts index 190024bb29..50496682e8 100644 --- a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts @@ -42,11 +42,11 @@ describe('NoForeignRootFieldsEntityPolicy', () => { }); it('works for the happy path', async () => { - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects unknown root fields', async () => { data.spec2 = {}; - await expect(policy.apply(data)).rejects.toThrow(/spec2/i); + await expect(policy.enforce(data)).rejects.toThrow(/spec2/i); }); }); diff --git a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts index 3c0a2f5d61..9d1851bc02 100644 --- a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts @@ -29,7 +29,7 @@ export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { this.knownFields = knownFields; } - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { for (const field of Object.keys(entity)) { if (!this.knownFields.includes(field)) { throw new Error(`Unknown field ${field}`); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts index 348eabbdac..8db33955a7 100644 --- a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts @@ -42,21 +42,23 @@ describe('ReservedFieldsEntityPolicy', () => { }); it('works for the happy path', async () => { - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects reserved keys in the spec root', async () => { data.spec.apiVersion = 'a/b'; - await expect(policy.apply(data)).rejects.toThrow(/spec.*apiVersion/i); + await expect(policy.enforce(data)).rejects.toThrow(/spec.*apiVersion/i); }); it('rejects reserved keys in labels', async () => { data.metadata.labels.apiVersion = 'a'; - await expect(policy.apply(data)).rejects.toThrow(/label.*apiVersion/i); + await expect(policy.enforce(data)).rejects.toThrow(/label.*apiVersion/i); }); it('rejects reserved keys in annotations', async () => { data.metadata.annotations.apiVersion = 'a'; - await expect(policy.apply(data)).rejects.toThrow(/annotation.*apiVersion/i); + await expect(policy.enforce(data)).rejects.toThrow( + /annotation.*apiVersion/i, + ); }); }); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts index be2f732ca4..d97469eecc 100644 --- a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts @@ -43,7 +43,7 @@ export class ReservedFieldsEntityPolicy implements EntityPolicy { ]; } - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { for (const field of this.reservedFields) { if (entity.spec?.hasOwnProperty(field)) { throw new Error( diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts index b9d1165a60..d24aee9fe6 100644 --- a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -43,7 +43,7 @@ describe('SchemaValidEntityPolicy', () => { }); it('works for the happy path', async () => { - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); // @@ -51,113 +51,113 @@ describe('SchemaValidEntityPolicy', () => { // it('rejects wrong root type', async () => { - await expect(policy.apply((7 as unknown) as Entity)).rejects.toThrow( + await expect(policy.enforce((7 as unknown) as Entity)).rejects.toThrow( /object/, ); }); it('rejects missing apiVersion', async () => { delete data.apiVersion; - await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); }); it('rejects bad apiVersion type', async () => { data.apiVersion = 7; - await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); }); it('rejects missing kind', async () => { delete data.kind; - await expect(policy.apply(data)).rejects.toThrow(/kind/); + await expect(policy.enforce(data)).rejects.toThrow(/kind/); }); it('rejects bad kind type', async () => { data.kind = 7; - await expect(policy.apply(data)).rejects.toThrow(/kind/); + await expect(policy.enforce(data)).rejects.toThrow(/kind/); }); // // metadata // - it('accepts missing metadata', async () => { - delete data.medatata; - await expect(policy.apply(data)).resolves.toBe(data); + it('rejects missing metadata', async () => { + delete data.metadata; + await expect(policy.enforce(data)).rejects.toThrow(/metadata/); }); it('rejects bad metadata type', async () => { data.metadata = 7; - await expect(policy.apply(data)).rejects.toThrow(/metadata/); + await expect(policy.enforce(data)).rejects.toThrow(/metadata/); }); it('accepts missing uid', async () => { delete data.metadata.uid; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad uid type', async () => { data.metadata.uid = 7; - await expect(policy.apply(data)).rejects.toThrow(/uid/); + await expect(policy.enforce(data)).rejects.toThrow(/uid/); }); it('accepts missing etag', async () => { delete data.metadata.etag; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad etag type', async () => { data.metadata.etag = 7; - await expect(policy.apply(data)).rejects.toThrow(/etag/); + await expect(policy.enforce(data)).rejects.toThrow(/etag/); }); it('accepts missing generation', async () => { delete data.metadata.generation; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad generation type', async () => { data.metadata.generation = 'a'; - await expect(policy.apply(data)).rejects.toThrow(/generation/); + await expect(policy.enforce(data)).rejects.toThrow(/generation/); }); it('accepts missing name', async () => { delete data.metadata.name; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad name type', async () => { data.metadata.name = 7; - await expect(policy.apply(data)).rejects.toThrow(/name/); + await expect(policy.enforce(data)).rejects.toThrow(/name/); }); it('accepts missing namespace', async () => { delete data.metadata.namespace; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad namespace type', async () => { data.metadata.namespace = 7; - await expect(policy.apply(data)).rejects.toThrow(/namespace/); + await expect(policy.enforce(data)).rejects.toThrow(/namespace/); }); it('accepts missing labels', async () => { delete data.metadata.labels; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad labels type', async () => { data.metadata.labels = 7; - await expect(policy.apply(data)).rejects.toThrow(/labels/); + await expect(policy.enforce(data)).rejects.toThrow(/labels/); }); it('accepts missing annotations', async () => { delete data.metadata.annotations; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad annotations type', async () => { data.metadata.annotations = 7; - await expect(policy.apply(data)).rejects.toThrow(/annotations/); + await expect(policy.enforce(data)).rejects.toThrow(/annotations/); }); // @@ -166,11 +166,11 @@ describe('SchemaValidEntityPolicy', () => { it('accepts missing spec', async () => { delete data.spec; - await expect(policy.apply(data)).resolves.toBe(data); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects non-object spec', async () => { data.spec = 7; - await expect(policy.apply(data)).rejects.toThrow(/spec/); + await expect(policy.enforce(data)).rejects.toThrow(/spec/); }); }); diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts index 7c0f5c20b6..3ae20a5094 100644 --- a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -70,7 +70,7 @@ export class SchemaValidEntityPolicy implements EntityPolicy { this.schema = schema; } - async apply(entity: Entity): Promise { + async enforce(entity: Entity): Promise { try { return await this.schema.validate(entity, { strict: true }); } catch (e) { diff --git a/packages/catalog-model/src/kinds/ComponentV1beta1.ts b/packages/catalog-model/src/kinds/ComponentV1beta1.ts index b0a3f627a5..b041bf7967 100644 --- a/packages/catalog-model/src/kinds/ComponentV1beta1.ts +++ b/packages/catalog-model/src/kinds/ComponentV1beta1.ts @@ -50,7 +50,7 @@ export class ComponentV1beta1Policy implements EntityPolicy { }); } - async apply(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { if ( envelope.apiVersion !== 'backstage.io/v1beta1' || envelope.kind !== 'Component' diff --git a/packages/catalog-model/src/setupTests.ts b/packages/catalog-model/src/setupTests.ts deleted file mode 100644 index f3b69cc361..0000000000 --- a/packages/catalog-model/src/setupTests.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 1d581cf23f..29ca8bdfa3 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -28,5 +28,5 @@ export type EntityPolicy = { * @returns The incoming entity, or a mutated version of the same * @throws An error if the entity should be rejected */ - apply(entity: Entity): Promise; + enforce(entity: Entity): Promise; }; diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index c8d0aec332..9a4297904a 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -32,14 +32,14 @@ describe('DatabaseManager', () => { readLocation: jest.fn(), }; const policy: EntityPolicy = { - apply: jest.fn(), + enforce: jest.fn(), }; await expect( DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(reader.readLocation).not.toHaveBeenCalled(); - expect(policy.apply).not.toHaveBeenCalled(); + expect(policy.enforce).not.toHaveBeenCalled(); }); it('can update a single location', async () => { @@ -70,7 +70,7 @@ describe('DatabaseManager', () => { ), }; const policy: EntityPolicy = { - apply: jest.fn(() => Promise.resolve(desc)), + enforce: jest.fn(() => Promise.resolve(desc)), }; await expect( @@ -118,7 +118,7 @@ describe('DatabaseManager', () => { ), }; const policy: EntityPolicy = { - apply: jest.fn(() => Promise.resolve(desc)), + enforce: jest.fn(() => Promise.resolve(desc)), }; await expect( @@ -170,7 +170,9 @@ describe('DatabaseManager', () => { ), }; const policy: EntityPolicy = { - apply: jest.fn(() => Promise.reject(new Error('parser error message'))), + enforce: jest.fn(() => + Promise.reject(new Error('parser error message')), + ), }; await expect( @@ -217,7 +219,9 @@ describe('DatabaseManager', () => { ), }; const policy: EntityPolicy = { - apply: jest.fn(() => Promise.reject(new Error('parser error message'))), + enforce: jest.fn(() => + Promise.reject(new Error('parser error message')), + ), }; await expect( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index a3aded18ec..c6e7a2e684 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -86,7 +86,7 @@ export class DatabaseManager { } try { - const entity = await entityPolicy.apply(readerItem.data); + const entity = await entityPolicy.enforce(readerItem.data); await DatabaseManager.refreshSingleEntity( database, location.id, diff --git a/plugins/catalog-backend/src/ingestion/IngestionModels.ts b/plugins/catalog-backend/src/ingestion/IngestionModels.ts index 616f1b49ae..def6f1fa5c 100644 --- a/plugins/catalog-backend/src/ingestion/IngestionModels.ts +++ b/plugins/catalog-backend/src/ingestion/IngestionModels.ts @@ -60,7 +60,7 @@ export class IngestionModels implements IngestionModel { result.push(item); } else { try { - const output = await this.entityPolicy.apply(item.data); + const output = await this.entityPolicy.enforce(item.data); result.push({ type: 'data', data: output }); } catch (e) { result.push({ type: 'error', error: e }); From dad0b8390cb5c23a9baec4ec47e1546c800cbcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 20:54:02 +0200 Subject: [PATCH 6/7] Update plugins/catalog-backend/src/ingestion/source/index.ts Co-authored-by: Ivan Shmidt --- plugins/catalog-backend/src/ingestion/source/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/source/index.ts b/plugins/catalog-backend/src/ingestion/source/index.ts index 3ed1063878..db7aa2f0bd 100644 --- a/plugins/catalog-backend/src/ingestion/source/index.ts +++ b/plugins/catalog-backend/src/ingestion/source/index.ts @@ -17,4 +17,4 @@ export { LocationReaders } from './LocationReaders'; export { FileLocationReader } from './readers/FileLocationReader'; export { GitHubLocationReader } from './readers/GitHubLocationReader'; -export { LocationReader } from './readers/types'; +export type { LocationReader } from './readers/types'; From 06879f55425747397fe4ce194efefd436a702245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 May 2020 21:01:43 +0200 Subject: [PATCH 7/7] Accidental premature test fix :) --- .../src/entity/policies/SchemaValidEntityPolicy.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts index d24aee9fe6..0837b75759 100644 --- a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -80,9 +80,9 @@ describe('SchemaValidEntityPolicy', () => { // metadata // - it('rejects missing metadata', async () => { + it('accepts missing metadata', async () => { delete data.metadata; - await expect(policy.enforce(data)).rejects.toThrow(/metadata/); + await expect(policy.enforce(data)).resolves.toBe(data); }); it('rejects bad metadata type', async () => {