diff --git a/packages/backend-common/src/setupTests.ts b/packages/backend-common/src/setupTests.ts index 3fa7cb04b4..a3b2b7e123 100644 --- a/packages/backend-common/src/setupTests.ts +++ b/packages/backend-common/src/setupTests.ts @@ -15,3 +15,4 @@ */ require('jest-fetch-mock').enableMocks(); +export {}; diff --git a/packages/backend/package.json b/packages/backend/package.json index 9aedafecbb..e4b77c1e41 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,7 +10,7 @@ }, "scripts": { "build": "tsc", - "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess nodemon", + "start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"nodemon -r esm\\\"", "lint": "backstage-cli lint", "test": "backstage-cli test", "clean": "backstage-cli clean", @@ -18,13 +18,15 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", + "@backstage/catalog-model": "^0.1.1-alpha.6", "@backstage/plugin-auth-backend": "^0.1.1-alpha.6", "@backstage/plugin-catalog-backend": "^0.1.1-alpha.6", - "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", - "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", "@backstage/plugin-identity-backend": "^0.1.1-alpha.6", + "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6", + "@backstage/plugin-sentry-backend": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", + "esm": "^3.2.25", "express": "^4.17.1", "helmet": "^3.22.0", "knex": "^0.21.1", diff --git a/packages/backend/src/index.test.ts b/packages/backend/src/index.test.ts index b3e2f19771..d18873c1f0 100644 --- a/packages/backend/src/index.test.ts +++ b/packages/backend/src/index.test.ts @@ -14,8 +14,11 @@ * limitations under the License. */ +import { PluginEnvironment } from './types'; + describe('test', () => { it('unbreaks the test runner', () => { - expect(true).toBeTruthy(); + const unbreaker = {} as PluginEnvironment; + expect(unbreaker).toBeTruthy(); }); }); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 687fd9157a..7e843cc80b 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -21,22 +21,28 @@ import { DatabaseManager, DescriptorParsers, LocationReaders, + IngestionModels, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; +import { EntityPolicies } from '@backstage/catalog-model'; export default async function ({ logger, database }: PluginEnvironment) { - const reader = LocationReaders.create(); - const parser = DescriptorParsers.create(); + const policy = new EntityPolicies(); + const ingestion = new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); const db = await DatabaseManager.createDatabase(database, logger); runPeriodically( - () => DatabaseManager.refreshLocations(db, reader, parser, logger), + () => DatabaseManager.refreshLocations(db, ingestion, policy, logger), 10000, ); const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db, reader); + const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion); return await createRouter({ entitiesCatalog, locationsCatalog, logger }); } diff --git a/packages/catalog-model/.eslintrc.js b/packages/catalog-model/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/packages/catalog-model/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/packages/catalog-model/README.md b/packages/catalog-model/README.md new file mode 100644 index 0000000000..755b9ee63c --- /dev/null +++ b/packages/catalog-model/README.md @@ -0,0 +1,12 @@ +# Catalog Model + +Contains the core model types and validators/policies used by the Backstage catalog functionality. + +This package will be imported both by the frontend and backend parts of the catalog, +as well as by others that want to consume catalog data. + +## Links + +- (Default frontend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog] +- (Default backend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog-backend] +- (The Backstage homepage)[https://backstage.io] diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json new file mode 100644 index 0000000000..ffd8fdba22 --- /dev/null +++ b/packages/catalog-model/package.json @@ -0,0 +1,34 @@ +{ + "name": "@backstage/catalog-model", + "version": "0.1.1-alpha.6", + "main": "dist/index.esm.js", + "main:src": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "lodash": "^4.17.15", + "yup": "^0.28.5" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.6", + "@types/jest": "^25.2.2", + "@types/lodash": "^4.14.151", + "@types/yup": "^0.28.2", + "yaml": "^1.9.2" + }, + "files": [ + "dist/**/*.{js,d.ts}" + ] +} diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts new file mode 100644 index 0000000000..fa25334d5f --- /dev/null +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + FieldFormatEntityPolicy, + NoForeignRootFieldsEntityPolicy, + ReservedFieldsEntityPolicy, + SchemaValidEntityPolicy, +} from './entity'; +import { ComponentV1beta1Policy } from './kinds'; +import { EntityPolicy } from './types'; + +// Helper that requires that all of a set of policies can be successfully +// applied +class AllEntityPolicies implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async enforce(entity: Entity): Promise { + let result = entity; + for (const policy of this.policies) { + result = await policy.enforce(entity); + } + return result; + } +} + +// Helper that requires that at least one of a set of policies can be +// successfully applied +class AnyEntityPolicy implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async enforce(entity: Entity): Promise { + for (const policy of this.policies) { + try { + return await policy.enforce(entity); + } catch { + continue; + } + } + throw new Error(`The entity did not match any known policy`); + } +} + +export class EntityPolicies implements EntityPolicy { + private readonly policy: EntityPolicy; + + static defaultPolicies(): EntityPolicy { + return EntityPolicies.allOf([ + EntityPolicies.allOf([ + new SchemaValidEntityPolicy(), + new NoForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy(), + new ReservedFieldsEntityPolicy(), + ]), + EntityPolicies.anyOf([new ComponentV1beta1Policy()]), + ]); + } + + static allOf(policies: EntityPolicy[]): EntityPolicy { + return new AllEntityPolicies(policies); + } + + static anyOf(policies: EntityPolicy[]): EntityPolicy { + return new AnyEntityPolicy(policies); + } + + constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) { + this.policy = policy; + } + + enforce(entity: Entity): Promise { + return this.policy.enforce(entity); + } +} diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts new file mode 100644 index 0000000000..80c5765ee1 --- /dev/null +++ b/packages/catalog-model/src/entity/Entity.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The format envelope that's common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type Entity = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Metadata related to the entity. + */ + metadata: EntityMeta; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Metadata fields common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type EntityMeta = { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace + kind pair. + */ + name: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: Record; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: Record; +}; diff --git a/packages/cli/src/index.test.ts b/packages/catalog-model/src/entity/index.ts similarity index 88% rename from packages/cli/src/index.test.ts rename to packages/catalog-model/src/entity/index.ts index 44f01f05e9..9e96021336 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -14,8 +14,5 @@ * limitations under the License. */ -describe('dummy', () => { - it('dummy', () => { - expect(1).toBe(1); - }); -}); +export type { Entity, EntityMeta } from './Entity'; +export * from './policies'; diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts new file mode 100644 index 0000000000..14b44108e5 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; + +describe('FieldFormatEntityPolicy', () => { + let data: any; + let policy: FieldFormatEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new FieldFormatEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad apiVersion', async () => { + data.apiVersion = 7; + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); + data.apiVersion = 'a#b'; + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad kind', async () => { + data.kind = 7; + await expect(policy.enforce(data)).rejects.toThrow(/kind/); + data.kind = 'a#b'; + await expect(policy.enforce(data)).rejects.toThrow(/kind/); + }); + + it('handles missing metadata gracefully', async () => { + delete data.medatata; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('handles missing spec gracefully', async () => { + delete data.spec; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad name', async () => { + data.metadata.name = 7; + await expect(policy.enforce(data)).rejects.toThrow(/name.*7/); + data.metadata.name = 'a'.repeat(1000); + await expect(policy.enforce(data)).rejects.toThrow(/name.*aaaa/); + }); + + it('rejects bad namespace', async () => { + data.metadata.namespace = 7; + await expect(policy.enforce(data)).rejects.toThrow(/namespace.*7/); + data.metadata.namespace = 'a'.repeat(1000); + await expect(policy.enforce(data)).rejects.toThrow(/namespace.*aaaa/); + }); + + it('rejects bad label key', async () => { + data.metadata.labels['a#b'] = 'value'; + await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad label value', async () => { + data.metadata.labels.a = 'a#b'; + await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad annotation key', async () => { + data.metadata.annotations['a#b'] = 'value'; + await expect(policy.enforce(data)).rejects.toThrow(/annotation.*a#b/i); + }); + + it('rejects bad annotation value', async () => { + data.metadata.annotations.a = 7; + await expect(policy.enforce(data)).rejects.toThrow(/annotation.*7/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts new file mode 100644 index 0000000000..4ccd8ec711 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { makeValidator, Validators } from '../../validation'; +import { Entity } from '../Entity'; + +/** + * Ensures that the format of individual fields of the entity envelope + * is valid. + * + * This does not take into account machine generated fields such as uid, etag + * and generation. + */ +export class FieldFormatEntityPolicy implements EntityPolicy { + private readonly validators: Validators; + + constructor(validators: Validators = makeValidator()) { + this.validators = validators; + } + + async enforce(entity: Entity): Promise { + function require( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + if (value === undefined || value === null) { + throw new Error(`${field} must have a value`); + } + + let isValid: boolean; + try { + isValid = validator(value); + } catch (e) { + throw new Error(`${field} could not be validated, ${e}`); + } + + if (!isValid) { + throw new Error(`${field} "${value}" is not valid`); + } + } + + function optional( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + return value === undefined || require(field, value, validator); + } + + require('apiVersion', entity.apiVersion, this.validators.isValidApiVersion); + require('kind', entity.kind, this.validators.isValidKind); + + require('metadata.name', entity.metadata.name, this.validators + .isValidEntityName); + optional( + 'metadata.namespace', + entity.metadata.namespace, + this.validators.isValidNamespace, + ); + + for (const [k, v] of Object.entries(entity.metadata.labels ?? [])) { + require(`labels.${k}`, k, this.validators.isValidLabelKey); + require(`labels.${k}`, v, this.validators.isValidLabelValue); + } + + for (const [k, v] of Object.entries(entity.metadata.annotations ?? [])) { + require(`annotations.${k}`, k, this.validators.isValidAnnotationKey); + require(`annotations.${k}`, v, this.validators.isValidAnnotationValue); + } + + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..50496682e8 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy'; + +describe('NoForeignRootFieldsEntityPolicy', () => { + let data: any; + let policy: NoForeignRootFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new NoForeignRootFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects unknown root fields', async () => { + data.spec2 = {}; + await expect(policy.enforce(data)).rejects.toThrow(/spec2/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts new file mode 100644 index 0000000000..9d1851bc02 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec']; + +/** + * Ensures that there are no foreign root fields in the entity. + */ +export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { + private readonly knownFields: string[]; + + constructor(knownFields: string[] = defaultKnownFields) { + this.knownFields = knownFields; + } + + async enforce(entity: Entity): Promise { + for (const field of Object.keys(entity)) { + if (!this.knownFields.includes(field)) { + throw new Error(`Unknown field ${field}`); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..8db33955a7 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; + +describe('ReservedFieldsEntityPolicy', () => { + let data: any; + let policy: ReservedFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ReservedFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects reserved keys in the spec root', async () => { + data.spec.apiVersion = 'a/b'; + await expect(policy.enforce(data)).rejects.toThrow(/spec.*apiVersion/i); + }); + + it('rejects reserved keys in labels', async () => { + data.metadata.labels.apiVersion = 'a'; + await expect(policy.enforce(data)).rejects.toThrow(/label.*apiVersion/i); + }); + + it('rejects reserved keys in annotations', async () => { + data.metadata.annotations.apiVersion = 'a'; + await expect(policy.enforce(data)).rejects.toThrow( + /annotation.*apiVersion/i, + ); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts new file mode 100644 index 0000000000..57029ffa24 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_RESERVED_ENTITY_FIELDS = [ + 'apiVersion', + 'kind', + 'uid', + 'etag', + 'generation', + 'name', + 'namespace', + 'labels', + 'annotations', + 'spec', +]; + +/** + * Ensures that fields are not given certain reserved names. + */ +export class ReservedFieldsEntityPolicy implements EntityPolicy { + private readonly reservedFields: string[]; + + constructor(fields?: string[]) { + this.reservedFields = [ + ...(fields ?? []), + ...DEFAULT_RESERVED_ENTITY_FIELDS, + ]; + } + + async enforce(entity: Entity): Promise { + for (const field of this.reservedFields) { + if (entity.spec?.hasOwnProperty(field)) { + throw new Error( + `The spec may not contain the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata.labels?.hasOwnProperty(field)) { + throw new Error( + `A label may not have the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata.annotations?.hasOwnProperty(field)) { + throw new Error( + `An annotation may not have the field ${field}, because it has reserved meaning`, + ); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts new file mode 100644 index 0000000000..c53fbf3e46 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { Entity } from '../Entity'; +import { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; + +describe('SchemaValidEntityPolicy', () => { + let data: any; + let policy: SchemaValidEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new SchemaValidEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + // + // apiVersion and kind + // + + it('rejects wrong root type', async () => { + await expect(policy.enforce((7 as unknown) as Entity)).rejects.toThrow( + /object/, + ); + }); + + it('rejects missing apiVersion', async () => { + delete data.apiVersion; + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad apiVersion type', async () => { + data.apiVersion = 7; + await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects missing kind', async () => { + delete data.kind; + await expect(policy.enforce(data)).rejects.toThrow(/kind/); + }); + + it('rejects bad kind type', async () => { + data.kind = 7; + await expect(policy.enforce(data)).rejects.toThrow(/kind/); + }); + + // + // metadata + // + + it('rejects missing metadata', async () => { + delete data.metadata; + await expect(policy.enforce(data)).rejects.toThrow(/metadata/); + }); + + it('rejects bad metadata type', async () => { + data.metadata = 7; + await expect(policy.enforce(data)).rejects.toThrow(/metadata/); + }); + + it('accepts missing uid', async () => { + delete data.metadata.uid; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad uid type', async () => { + data.metadata.uid = 7; + await expect(policy.enforce(data)).rejects.toThrow(/uid/); + }); + + it('accepts missing etag', async () => { + delete data.metadata.etag; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad etag type', async () => { + data.metadata.etag = 7; + await expect(policy.enforce(data)).rejects.toThrow(/etag/); + }); + + it('accepts missing generation', async () => { + delete data.metadata.generation; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad generation type', async () => { + data.metadata.generation = 'a'; + await expect(policy.enforce(data)).rejects.toThrow(/generation/); + }); + + it('rejects missing name', async () => { + delete data.metadata.name; + await expect(policy.enforce(data)).rejects.toThrow(/name/); + }); + + it('rejects bad name type', async () => { + data.metadata.name = 7; + await expect(policy.enforce(data)).rejects.toThrow(/name/); + }); + + it('accepts missing namespace', async () => { + delete data.metadata.namespace; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad namespace type', async () => { + data.metadata.namespace = 7; + await expect(policy.enforce(data)).rejects.toThrow(/namespace/); + }); + + it('accepts missing labels', async () => { + delete data.metadata.labels; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad labels type', async () => { + data.metadata.labels = 7; + await expect(policy.enforce(data)).rejects.toThrow(/labels/); + }); + + it('accepts missing annotations', async () => { + delete data.metadata.annotations; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects bad annotations type', async () => { + data.metadata.annotations = 7; + await expect(policy.enforce(data)).rejects.toThrow(/annotations/); + }); + + // + // spec + // + + it('accepts missing spec', async () => { + delete data.spec; + await expect(policy.enforce(data)).resolves.toBe(data); + }); + + it('rejects non-object spec', async () => { + data.spec = 7; + await expect(policy.enforce(data)).rejects.toThrow(/spec/); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts new file mode 100644 index 0000000000..d367022ff1 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_ENTITY_SCHEMA = yup.object({ + apiVersion: yup.string().required(), + kind: yup.string().required(), + metadata: yup + .object({ + uid: yup + .string() + .notRequired() + .test( + 'metadata.uid', + 'The uid must not be empty', + value => value === undefined || value.length > 0, + ), + etag: yup + .string() + .notRequired() + .test( + 'metadata.etag', + 'The etag must not be empty', + value => value === undefined || value.length > 0, + ), + generation: yup + .number() + .notRequired() + .test( + 'metadata.generation', + 'The generation must be an integer greater than zero', + value => value === undefined || (value === (value | 0) && value > 0), + ), + name: yup.string().required(), + namespace: yup.string().notRequired(), + labels: yup.object>().notRequired(), + annotations: yup.object>().notRequired(), + }) + .required(), + spec: yup.object({}).notRequired(), +}); + +/** + * Ensures that the entity spec is valid according to a schema. + * + * This should be the first policy in the list, to ensure that other downstream + * policies can work with a structure that is at least valid in therms of the + * typescript type. + */ +export class SchemaValidEntityPolicy implements EntityPolicy { + private readonly schema: yup.Schema; + + constructor(schema: yup.Schema = DEFAULT_ENTITY_SCHEMA) { + this.schema = schema; + } + + async enforce(entity: Entity): Promise { + try { + return await this.schema.validate(entity, { strict: true }); + } catch (e) { + throw new Error(`Malformed envelope, ${e}`); + } + } +} diff --git a/packages/core/src/hooks/useSidebarPinState.ts b/packages/catalog-model/src/entity/policies/index.ts similarity index 66% rename from packages/core/src/hooks/useSidebarPinState.ts rename to packages/catalog-model/src/entity/policies/index.ts index bbe548c699..d64053f7cb 100644 --- a/packages/core/src/hooks/useSidebarPinState.ts +++ b/packages/catalog-model/src/entity/policies/index.ts @@ -13,16 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { useContext } from 'react'; -import { SidebarPinStateContext } from '../layout/Sidebar'; -export function useSidebarPinState() { - const { isPinned, toggleSidebarPinState } = useContext( - SidebarPinStateContext, - ); - - return { - isPinned, - toggleSidebarPinState, - }; -} +export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; +export { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy'; +export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; +export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; diff --git a/plugins/catalog-backend/src/validation/index.ts b/packages/catalog-model/src/index.ts similarity index 77% rename from plugins/catalog-backend/src/validation/index.ts rename to packages/catalog-model/src/index.ts index be607e43ec..fb51461053 100644 --- a/plugins/catalog-backend/src/validation/index.ts +++ b/packages/catalog-model/src/index.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -export * from './CommonValidatorFunctions'; -export * from './KubernetesValidatorFunctions'; -export * from './makeValidator'; -export * from './types'; +export * from './entity'; +export { EntityPolicies } from './EntityPolicies'; +export * from './kinds'; +export type { EntityPolicy } from './types'; +export * from './validation'; diff --git a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts b/packages/catalog-model/src/kinds/ComponentV1beta1.ts similarity index 62% rename from plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts rename to packages/catalog-model/src/kinds/ComponentV1beta1.ts index 974b34aa8e..b041bf7967 100644 --- a/plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts +++ b/packages/catalog-model/src/kinds/ComponentV1beta1.ts @@ -15,19 +15,28 @@ */ import * as yup from 'yup'; -import { DescriptorEnvelope, KindParser, ParserError } from '../types'; +import type { Entity, EntityMeta } from '../entity/Entity'; +import type { EntityPolicy } from '../types'; -export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope { +const API_VERSION = 'backstage.io/v1beta1'; +const KIND = 'Component'; + +export interface ComponentV1beta1 extends Entity { + apiVersion: typeof API_VERSION; + kind: typeof KIND; + metadata: EntityMeta & { + name: string; + }; spec: { type: string; }; } -export class ComponentDescriptorV1beta1Parser implements KindParser { +export class ComponentV1beta1Policy implements EntityPolicy { private schema: yup.Schema; constructor() { - this.schema = yup.object>({ + this.schema = yup.object>({ metadata: yup .object({ name: yup.string().required(), @@ -41,23 +50,14 @@ export class ComponentDescriptorV1beta1Parser implements KindParser { }); } - async tryParse( - envelope: DescriptorEnvelope, - ): Promise { + async enforce(envelope: Entity): Promise { if ( envelope.apiVersion !== 'backstage.io/v1beta1' || envelope.kind !== 'Component' ) { - return undefined; + throw new Error('Unsupported apiVersion / kind'); } - try { - return await this.schema.validate(envelope, { strict: true }); - } catch (e) { - throw new ParserError( - `Malformed component, ${e}`, - envelope.metadata?.name, - ); - } + return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts new file mode 100644 index 0000000000..97d22c14a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ComponentV1beta1 } from './ComponentV1beta1'; +export { ComponentV1beta1Policy } from './ComponentV1beta1'; +export { ComponentV1beta1 as Component }; +export { ComponentV1beta1 }; diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts new file mode 100644 index 0000000000..29ca8bdfa3 --- /dev/null +++ b/packages/catalog-model/src/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from './entity/Entity'; + +/** + * A policy for validation or mutation to be applied to entities as they are + * entering the system. + */ +export type EntityPolicy = { + /** + * Applies validation or mutation on an entity. + * + * @param entity The entity, as validated/mutated so far in the policy tree + * @returns The incoming entity, or a mutated version of the same + * @throws An error if the entity should be rejected + */ + enforce(entity: Entity): Promise; +}; diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts similarity index 100% rename from plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts rename to packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts diff --git a/plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts similarity index 100% rename from plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts rename to packages/catalog-model/src/validation/CommonValidatorFunctions.ts diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts similarity index 100% rename from plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts rename to packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts diff --git a/plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts similarity index 100% rename from plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts rename to packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts diff --git a/packages/catalog-model/src/validation/index.ts b/packages/catalog-model/src/validation/index.ts new file mode 100644 index 0000000000..d679a5323c --- /dev/null +++ b/packages/catalog-model/src/validation/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CommonValidatorFunctions } from './CommonValidatorFunctions'; +export { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +export { makeValidator } from './makeValidator'; +export type { Validators } from './types'; diff --git a/plugins/catalog-backend/src/validation/makeValidator.ts b/packages/catalog-model/src/validation/makeValidator.ts similarity index 100% rename from plugins/catalog-backend/src/validation/makeValidator.ts rename to packages/catalog-model/src/validation/makeValidator.ts diff --git a/plugins/catalog-backend/src/validation/types.ts b/packages/catalog-model/src/validation/types.ts similarity index 100% rename from plugins/catalog-backend/src/validation/types.ts rename to packages/catalog-model/src/validation/types.ts diff --git a/packages/core/src/api/createApp.tsx b/packages/core/src/api-wrappers/createApp.tsx similarity index 100% rename from packages/core/src/api/createApp.tsx rename to packages/core/src/api-wrappers/createApp.tsx diff --git a/packages/core/src/api/index.ts b/packages/core/src/api-wrappers/index.ts similarity index 100% rename from packages/core/src/api/index.ts rename to packages/core/src/api-wrappers/index.ts diff --git a/packages/core/src/components/CopyTextButton/CopyTextButton.stories.tsx b/packages/core/src/components/CopyTextButton/CopyTextButton.stories.tsx index 3dc5854ab3..92db2073d3 100644 --- a/packages/core/src/components/CopyTextButton/CopyTextButton.stories.tsx +++ b/packages/core/src/components/CopyTextButton/CopyTextButton.stories.tsx @@ -16,33 +16,10 @@ import React from 'react'; import CopyTextButton from '.'; -import { - ApiProvider, - errorApiRef, - ApiRegistry, - ErrorApi, -} from '@backstage/core-api'; export default { title: 'CopyTextButton', component: CopyTextButton, - decorators: [ - (storyFn: () => JSX.Element) => { - // TODO: move this to common storybook config, requires core package to be separate from components - const registry = ApiRegistry.from([ - [ - errorApiRef, - { - post(error) { - // eslint-disable-next-line no-alert - window.alert(`Component posted error, ${error}`); - }, - } as ErrorApi, - ], - ]); - return ; - }, - ], }; export const Default = () => ( diff --git a/packages/core/src/icons/icons.tsx b/packages/core/src/icons/icons.tsx deleted file mode 100644 index e8834a6162..0000000000 --- a/packages/core/src/icons/icons.tsx +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { SvgIconProps } from '@material-ui/core'; -import PeopleIcon from '@material-ui/icons/People'; -import PersonIcon from '@material-ui/icons/Person'; -import React, { FC } from 'react'; -import { useApp } from '@backstage/core-api'; -import { IconComponent, SystemIconKey, SystemIcons } from './types'; - -export const defaultSystemIcons: SystemIcons = { - user: PersonIcon, - group: PeopleIcon, -}; - -const overridableSystemIcon = (key: SystemIconKey): IconComponent => { - const Component: FC = props => { - const app = useApp(); - const Icon = app.getSystemIcon(key); - return ; - }; - return Component; -}; - -export const UserIcon = overridableSystemIcon('user'); -export const GroupIcon = overridableSystemIcon('group'); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 37f01eb3f4..2ceea470f4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,7 +16,7 @@ export * from '@backstage/core-api'; -export * from './api'; +export * from './api-wrappers'; export * from './layout'; export { default as CodeSnippet } from './components/CodeSnippet'; @@ -39,4 +39,3 @@ export { default as TrendLine } from './components/TrendLine'; export { FeatureCalloutCircular } from './components/FeatureDiscovery/FeatureCalloutCircular'; export * from './components/Status'; export { default as WarningPanel } from './components/WarningPanel'; -export type { IconComponent } from './icons'; diff --git a/packages/core/src/layout/Sidebar/Bar.tsx b/packages/core/src/layout/Sidebar/Bar.tsx index c5a8ae5669..1aadf9fd3e 100644 --- a/packages/core/src/layout/Sidebar/Bar.tsx +++ b/packages/core/src/layout/Sidebar/Bar.tsx @@ -16,10 +16,10 @@ import { makeStyles } from '@material-ui/core'; import clsx from 'clsx'; -import React, { FC, useRef, useState } from 'react'; +import React, { FC, useRef, useState, useContext } from 'react'; import { sidebarConfig, SidebarContext } from './config'; import { BackstageTheme } from '@backstage/theme'; -import { useSidebarPinState } from '../../hooks/useSidebarPinState'; +import { SidebarPinStateContext } from './Page'; const useStyles = makeStyles(theme => ({ root: { @@ -76,7 +76,7 @@ export const Sidebar: FC = ({ const classes = useStyles(); const [state, setState] = useState(State.Closed); const hoverTimerRef = useRef(); - const { isPinned } = useSidebarPinState(); + const { isPinned } = useContext(SidebarPinStateContext); const handleOpen = () => { if (isPinned) { diff --git a/packages/core/src/layout/Sidebar/Items.tsx b/packages/core/src/layout/Sidebar/Items.tsx index 4696f53a37..ddd71cc189 100644 --- a/packages/core/src/layout/Sidebar/Items.tsx +++ b/packages/core/src/layout/Sidebar/Items.tsx @@ -22,12 +22,12 @@ import { Typography, Badge, } from '@material-ui/core'; +import { IconComponent } from '@backstage/core-api'; import SearchIcon from '@material-ui/icons/Search'; import clsx from 'clsx'; import React, { FC, useContext, useState, KeyboardEventHandler } from 'react'; import { NavLink } from 'react-router-dom'; import { sidebarConfig, SidebarContext } from './config'; -import { IconComponent } from '../../icons'; const useStyles = makeStyles(theme => { const { diff --git a/packages/core/src/layout/Sidebar/Page.tsx b/packages/core/src/layout/Sidebar/Page.tsx index 750f764c2d..2a4cf0fbb2 100644 --- a/packages/core/src/layout/Sidebar/Page.tsx +++ b/packages/core/src/layout/Sidebar/Page.tsx @@ -18,7 +18,7 @@ import { makeStyles } from '@material-ui/core'; import React, { createContext, FC, useEffect, useState } from 'react'; import { sidebarConfig } from './config'; import { BackstageTheme } from '@backstage/theme'; -import { LocalStorage } from '../../data/localStorage'; +import { LocalStorage } from './localStorage'; const useStyles = makeStyles({ root: { diff --git a/packages/core/src/data/localStorage.test.ts b/packages/core/src/layout/Sidebar/localStorage.test.ts similarity index 100% rename from packages/core/src/data/localStorage.test.ts rename to packages/core/src/layout/Sidebar/localStorage.test.ts diff --git a/packages/core/src/data/localStorage.ts b/packages/core/src/layout/Sidebar/localStorage.ts similarity index 100% rename from packages/core/src/data/localStorage.ts rename to packages/core/src/layout/Sidebar/localStorage.ts diff --git a/packages/storybook/.storybook/apis.js b/packages/storybook/.storybook/apis.js new file mode 100644 index 0000000000..5743dac50b --- /dev/null +++ b/packages/storybook/.storybook/apis.js @@ -0,0 +1,16 @@ +import { + ApiRegistry, + alertApiRef, + errorApiRef, + AlertApiForwarder, + ErrorApiForwarder, + ErrorAlerter, +} from '@backstage/core'; + +const builder = ApiRegistry.builder(); + +const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); + +builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); + +export const apis = builder.build(); diff --git a/packages/storybook/.storybook/config.js b/packages/storybook/.storybook/config.js index 941b02718b..69145ea057 100644 --- a/packages/storybook/.storybook/config.js +++ b/packages/storybook/.storybook/config.js @@ -3,14 +3,18 @@ import { addDecorator, addParameters } from '@storybook/react'; import { lightTheme, darkTheme } from '@backstage/theme'; import { CssBaseline, ThemeProvider } from '@material-ui/core'; import { useDarkMode } from 'storybook-dark-mode'; -import { Content } from '@backstage/core'; +import { Content, ApiProvider, AlertDisplay } from '@backstage/core'; +import { apis } from './apis'; -addDecorator((story) => ( - - - {story()} - - +addDecorator(story => ( + + + + + {story()} + + + )); addParameters({ diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index fe82e544e7..9cb8017f15 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -27,14 +27,10 @@ "yn": "^4.0.0", "passport": "^0.4.1", "passport-google-oauth20": "^2.0.0", - "passport-oauth2-refresh": "^2.0.0", - "passport-oauth2": "^1.5.0", "cookie-parser": "^1.4.5", - "@types/passport-oauth2-refresh": "^1.1.1", "@types/passport": "^1.0.3", "@types/passport-google-oauth20": "^2.0.3", - "@types/cookie-parser": "^1.4.2", - "@types/passport-oauth2": "^1.4.9" + "@types/cookie-parser": "^1.4.2" }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.6", diff --git a/plugins/auth-backend/src/providers/OAuthProvider.ts b/plugins/auth-backend/src/providers/OAuthProvider.ts new file mode 100644 index 0000000000..81f5ed25a6 --- /dev/null +++ b/plugins/auth-backend/src/providers/OAuthProvider.ts @@ -0,0 +1,193 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express, { CookieOptions } from 'express'; +import crypto from 'crypto'; +import { AuthProviderRouteHandlers, OAuthProviderHandlers } from './types'; +import { InputError } from '@backstage/backend-common'; +import { postMessageResponse, ensuresXRequestedWith } from './utils'; + +export const THOUSAND_DAYS_MS = 1000 * 24 * 60 * 60 * 1000; +export const TEN_MINUTES_MS = 600 * 1000; + +export const verifyNonce = (req: express.Request, provider: string) => { + const cookieNonce = req.cookies[`${provider}-nonce`]; + const stateNonce = req.query.state; + + if (!cookieNonce || !stateNonce) { + throw new Error('Missing nonce'); + } + + if (cookieNonce !== stateNonce) { + throw new Error('Invalid nonce'); + } +}; + +export const setNonceCookie = (res: express.Response, provider: string) => { + const nonce = crypto.randomBytes(16).toString('base64'); + + const options: CookieOptions = { + maxAge: TEN_MINUTES_MS, + secure: false, + sameSite: 'none', + domain: 'localhost', + path: `/auth/${provider}/handler`, + httpOnly: true, + }; + + res.cookie(`${provider}-nonce`, nonce, options); + + return nonce; +}; + +export const setRefreshTokenCookie = ( + res: express.Response, + provider: string, + refreshToken: string, +) => { + const options: CookieOptions = { + maxAge: THOUSAND_DAYS_MS, + secure: false, + sameSite: 'none', + domain: 'localhost', + path: `/auth/${provider}`, + httpOnly: true, + }; + + res.cookie(`${provider}-refresh-token`, refreshToken, options); +}; + +export const removeRefreshTokenCookie = ( + res: express.Response, + provider: string, +) => { + const options: CookieOptions = { + maxAge: 0, + secure: false, + sameSite: 'none', + domain: 'localhost', + path: `/auth/${provider}`, + httpOnly: true, + }; + + res.cookie(`${provider}-refresh-token`, '', options); +}; + +export class OAuthProvider implements AuthProviderRouteHandlers { + private readonly provider: string; + private readonly providerHandlers: OAuthProviderHandlers; + constructor(providerHandlers: OAuthProviderHandlers, provider: string) { + this.provider = provider; + this.providerHandlers = providerHandlers; + } + + async start(req: express.Request, res: express.Response): Promise { + // retrieve scopes from request + const scope = req.query.scope?.toString() ?? ''; + + if (!scope) { + throw new InputError('missing scope parameter'); + } + + // set a nonce cookie before redirecting to oauth provider + const nonce = setNonceCookie(res, this.provider); + + const options = { + scope, + accessType: 'offline', + prompt: 'consent', + state: nonce, + }; + const { url, status } = await this.providerHandlers.start(req, options); + + res.statusCode = status || 302; + res.setHeader('Location', url); + res.setHeader('Content-Length', '0'); + res.end(); + } + + async frameHandler( + req: express.Request, + res: express.Response, + ): Promise { + try { + // verify nonce cookie and state cookie on callback + verifyNonce(req, this.provider); + + const { user, info } = await this.providerHandlers.handler(req); + + // throw error if missing refresh token + const { refreshToken } = info; + if (!refreshToken) { + throw new Error('Missing refresh token'); + } + + // set new refresh token + setRefreshTokenCookie(res, this.provider, refreshToken); + + // post message back to popup if successful + return postMessageResponse(res, { + type: 'auth-result', + payload: user, + }); + } catch (error) { + // post error message back to popup if failure + return postMessageResponse(res, { + type: 'auth-result', + error: { + name: error.name, + message: error.message, + }, + }); + } + } + + async logout(req: express.Request, res: express.Response): Promise { + if (!ensuresXRequestedWith(req)) { + return res.status(401).send('Invalid X-Requested-With header'); + } + + // remove refresh token cookie before logout + removeRefreshTokenCookie(res, this.provider); + return res.send('logout!'); + } + + async refresh(req: express.Request, res: express.Response): Promise { + if (!ensuresXRequestedWith(req)) { + return res.status(401).send('Invalid X-Requested-With header'); + } + + try { + const refreshToken = req.cookies[`${this.provider}-refresh-token`]; + + // throw error if refresh token is missing in the request + if (!refreshToken) { + throw new Error('Missing session cookie'); + } + + const scope = req.query.scope?.toString() ?? ''; + + // get new access_token + const refreshInfo = await this.providerHandlers.refresh( + refreshToken, + scope, + ); + return res.send(refreshInfo); + } catch (error) { + return res.status(401).send(`${error.message}`); + } + } +} diff --git a/plugins/auth-backend/src/providers/PassportStrategyHelper.ts b/plugins/auth-backend/src/providers/PassportStrategyHelper.ts new file mode 100644 index 0000000000..3ec8539330 --- /dev/null +++ b/plugins/auth-backend/src/providers/PassportStrategyHelper.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import passport from 'passport'; +import { RedirectInfo, RefreshTokenResponse } from './types'; + +export const executeRedirectStrategy = async ( + req: express.Request, + providerStrategy: passport.Strategy, + options: any, +): Promise => { + return new Promise(resolve => { + const strategy = Object.create(providerStrategy); + strategy.redirect = (url: string, status?: number) => { + resolve({ url, status: status ?? undefined }); + }; + + strategy.authenticate(req, { ...options }); + }); +}; + +export const executeFrameHandlerStrategy = async ( + req: express.Request, + providerStrategy: passport.Strategy, +) => { + return new Promise<{ user: any; info: any }>((resolve, reject) => { + const strategy = Object.create(providerStrategy); + strategy.success = (user: any, info: any) => { + resolve({ user, info }); + }; + strategy.fail = ( + info: { type: 'success' | 'error'; message?: string }, + // _status: number, + ) => { + reject(new Error(`Authentication rejected, ${info.message ?? ''}`)); + }; + strategy.error = (error: Error) => { + reject(new Error(`Authentication failed, ${error}`)); + }; + strategy.redirect = () => { + reject(new Error('Unexpected redirect')); + }; + + strategy.authenticate(req); + }); +}; + +export const executeRefreshTokenStrategy = async ( + providerstrategy: passport.Strategy, + refreshToken: string, + scope: string, +): Promise => { + return new Promise((resolve, reject) => { + const anyStrategy = providerstrategy as any; + const OAuth2 = anyStrategy._oauth2.constructor; + const oauth2 = new OAuth2( + anyStrategy._oauth2._clientId, + anyStrategy._oauth2._clientSecret, + anyStrategy._oauth2._baseSite, + anyStrategy._oauth2._authorizeUrl, + anyStrategy._refreshURL || anyStrategy._oauth2._accessTokenUrl, + anyStrategy._oauth2._customHeaders, + ); + + oauth2.getOAuthAccessToken( + refreshToken, + { + scope, + grant_type: 'refresh_token', + }, + ( + err: Error | null, + accessToken: string, + _refreshToken: string, + params: any, + ) => { + if (err) { + reject(new Error(`Failed to refresh access token ${err}`)); + } + if (!accessToken) { + reject( + new Error( + `Failed to refresh access token, no access token received`, + ), + ); + } + resolve({ + accessToken, + params, + }); + }, + ); + }); +}; diff --git a/plugins/auth-backend/src/providers/factories.test.ts b/plugins/auth-backend/src/providers/factories.test.ts deleted file mode 100644 index 1647f62682..0000000000 --- a/plugins/auth-backend/src/providers/factories.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import express from 'express'; -import passport from 'passport'; -import { AuthProvider, AuthProviderRouteHandlers } from './types'; -import { ProviderFactories } from './factories'; - -class MyAuthProvider implements AuthProvider, AuthProviderRouteHandlers { - strategy(): passport.Strategy { - return new passport.Strategy(); - } - async start(_: express.Request, res: express.Response): Promise { - res.send('start'); - } - async frameHandler(_: express.Request, res: express.Response): Promise { - res.send('frameHandler'); - } - async logout(_: express.Request, res: express.Response): Promise { - res.send('logout'); - } -} - -describe('getProviderFactory', () => { - it('makes a provider for MyAuthProvider', () => { - jest - .spyOn(ProviderFactories, 'getProviderFactory') - .mockReturnValueOnce(MyAuthProvider); - const provider = ProviderFactories.getProviderFactory('a'); - expect(provider).toBeDefined(); - }); - - it('throws an error when provider implementation does not exist', () => { - expect(() => { - ProviderFactories.getProviderFactory('b'); - }).toThrow('Provider Implementation missing for : b auth provider'); - }); -}); diff --git a/plugins/auth-backend/src/providers/factories.ts b/plugins/auth-backend/src/providers/factories.ts index 077d45076e..0a1e639082 100644 --- a/plugins/auth-backend/src/providers/factories.ts +++ b/plugins/auth-backend/src/providers/factories.ts @@ -15,7 +15,7 @@ */ import { AuthProviderFactories, AuthProviderFactory } from './types'; -import { GoogleAuthProvider } from './google/provider'; +import { GoogleAuthProvider } from './google'; export class ProviderFactories { private static readonly providerFactories: AuthProviderFactories = { diff --git a/packages/core/src/icons/index.ts b/plugins/auth-backend/src/providers/google/index.ts similarity index 92% rename from packages/core/src/icons/index.ts rename to plugins/auth-backend/src/providers/google/index.ts index 4c97d27176..0ec98bef89 100644 --- a/packages/core/src/icons/index.ts +++ b/plugins/auth-backend/src/providers/google/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ -export * from './icons'; -export * from './types'; +export { GoogleAuthProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/google/provider.test.ts b/plugins/auth-backend/src/providers/google/provider.test.ts deleted file mode 100644 index 327bf0260b..0000000000 --- a/plugins/auth-backend/src/providers/google/provider.test.ts +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - GoogleAuthProvider, - THOUSAND_DAYS_MS, - TEN_MINUTES_MS, -} from './provider'; -import passport from 'passport'; -import express from 'express'; -import * as utils from './../utils'; -import refresh from 'passport-oauth2-refresh'; - -const googleAuthProviderConfig = { - provider: 'google', - options: { - clientID: 'a', - clientSecret: 'b', - callbackURL: 'c', - }, -}; - -const googleAuthProviderConfigInvalidOptions = { - provider: 'google', - options: {}, -}; - -describe('GoogleAuthProvider', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - describe('create a new provider', () => { - it('should succeed with valid config', () => { - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - expect(googleAuthProvider).toBeDefined(); - expect(googleAuthProvider.start).toBeDefined(); - expect(googleAuthProvider.logout).toBeDefined(); - expect(googleAuthProvider.frameHandler).toBeDefined(); - expect(googleAuthProvider.strategy).toBeDefined(); - }); - }); - - describe('start authentication handler', () => { - const mockResponse = ({ - send: jest.fn().mockReturnThis(), - status: jest.fn().mockReturnThis(), - cookie: jest.fn().mockReturnThis(), - } as unknown) as express.Response; - const mockNext: express.NextFunction = jest.fn(); - - it('should initiate authenticate request with provided scopes', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - query: { - scope: 'a,b', - }, - } as unknown) as express.Request; - - const spyPassport = jest - .spyOn(passport, 'authenticate') - .mockImplementation(() => jest.fn()); - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - googleAuthProvider.start(mockRequest, mockResponse, mockNext); - expect(spyPassport).toBeCalledTimes(1); - expect(spyPassport).toBeCalledWith('google', { - scope: 'a,b', - accessType: 'offline', - prompt: 'consent', - state: expect.any(String), - }); - }); - - it('should set a nonce cookie', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - query: { - scope: 'a,b', - }, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - googleAuthProvider.start(mockRequest, mockResponse, mockNext); - expect(mockResponse.cookie).toBeCalledTimes(1); - expect(mockResponse.cookie).toBeCalledWith( - 'google-nonce', - expect.any(String), - expect.objectContaining({ - maxAge: TEN_MINUTES_MS, - path: `/auth/${googleAuthProviderConfig.provider}/handler`, - }), - ); - }); - - it('should throw error if no scopes provided', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - query: {}, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - expect(() => { - googleAuthProvider.start(mockRequest, mockResponse, mockNext); - }).toThrowError('missing scope parameter'); - }); - }); - - describe('logout handler', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - } as unknown) as express.Request; - - it('should perform logout and respond with 200', () => { - const mockResponse: any = ({ - send: jest.fn(), - cookie: jest.fn(), - } as unknown) as express.Response; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - const spyResponse = jest - .spyOn(mockResponse, 'send') - .mockImplementation(() => jest.fn()); - - googleAuthProvider.logout(mockRequest, mockResponse); - expect(spyResponse).toBeCalledTimes(1); - expect(spyResponse).toBeCalledWith('logout!'); - expect(mockResponse.cookie).toBeCalledTimes(1); - expect(mockResponse.cookie).toBeCalledWith( - 'google-refresh-token', - '', - expect.objectContaining({ maxAge: 0 }), - ); - }); - }); - - describe('redirect frame handler', () => { - const mockResponse: any = ({ - status: jest.fn().mockReturnThis(), - send: jest.fn().mockReturnThis(), - cookie: jest.fn().mockReturnThis(), - } as unknown) as express.Response; - const mockNext: express.NextFunction = jest.fn(); - - it('should call authenticate and post a response', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-nonce': 'NONCE' }, - query: { - state: 'NONCE', - }, - } as unknown) as express.Request; - - const spyPostMessage = jest - .spyOn(utils, 'postMessageResponse') - .mockImplementation(() => jest.fn()); - - const spyPassport = jest - .spyOn(passport, 'authenticate') - .mockImplementation((_x, callbackFunc) => { - const cb = callbackFunc as Function; - cb(null, { refreshToken: 'REFRESH_TOKEN' }); - return jest.fn(); - }); - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(spyPassport).toBeCalledTimes(1); - expect(spyPostMessage).toBeCalledTimes(1); - expect(mockResponse.cookie).toBeCalledTimes(1); - expect(mockResponse.cookie).toBeCalledWith( - 'google-refresh-token', - 'REFRESH_TOKEN', - expect.objectContaining({ - path: '/auth/google', - sameSite: 'none', - httpOnly: true, - maxAge: THOUSAND_DAYS_MS, - }), - ); - }); - - it('should respond with a error message if no refresh token returned', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-nonce': 'NONCE' }, - query: { - state: 'NONCE', - }, - } as unknown) as express.Request; - - const spyPassport = jest - .spyOn(passport, 'authenticate') - .mockImplementation((_x, callbackFunc) => { - const cb = callbackFunc as Function; - cb(null, {}); - return jest.fn(); - }); - - const spyPostMessage = jest - .spyOn(utils, 'postMessageResponse') - .mockImplementation(() => jest.fn()); - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(spyPassport).toBeCalledTimes(1); - expect(spyPostMessage).toBeCalledTimes(1); - expect(spyPostMessage).toBeCalledWith(mockResponse, { - type: 'auth-result', - error: new Error('Missing refresh token'), - }); - }); - - it('should respond with a error message if auth failed', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-nonce': 'NONCE' }, - query: { - state: 'NONCE', - }, - } as unknown) as express.Request; - - const spyPassport = jest - .spyOn(passport, 'authenticate') - .mockImplementation((_x, callbackFunc) => { - const cb = callbackFunc as Function; - cb(new Error('TokenError'), null); - return jest.fn(); - }); - - const spyPostMessage = jest - .spyOn(utils, 'postMessageResponse') - .mockImplementation(() => jest.fn()); - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(spyPassport).toBeCalledTimes(1); - expect(spyPostMessage).toBeCalledTimes(1); - expect(spyPostMessage).toBeCalledWith(mockResponse, { - type: 'auth-result', - error: new Error('Google auth failed, Error: TokenError'), - }); - }); - - it('should respond with a error message if cookie nonce is missing', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: {}, - query: { state: 'NONCE' }, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith('Missing nonce'); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - }); - - it('should respond with a error message if state nonce is missing', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-nonce': 'NONCE' }, - query: {}, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith('Missing nonce'); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - }); - - it('should respond with a error message if nonce mismatch', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-nonce': 'NONCA' }, - query: { state: 'NONCEB' }, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.frameHandler(mockRequest, mockResponse, mockNext); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith('Invalid nonce'); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - }); - }); - - describe('strategy handler', () => { - it('should return a valid passport strategy', () => { - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - expect(googleAuthProvider.strategy()).toBeInstanceOf(passport.Strategy); - }); - - it('should throw an error for invalid options', () => { - expect(() => { - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfigInvalidOptions, - ); - googleAuthProvider.strategy(); - }).toThrow(); - }); - }); - - describe('refresh token handler', () => { - const mockResponse = ({ - status: jest.fn().mockReturnThis(), - send: jest.fn().mockReturnThis(), - } as unknown) as express.Response; - - describe('no refresh token cookie', () => { - it('should respond with a 401', () => { - const mockRequest = ({ - cookies: jest.fn(), - header: () => 'XMLHttpRequest', - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - googleAuthProvider.refresh(mockRequest, mockResponse); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith('Missing session cookie'); - - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - }); - }); - - describe('refresh token cookie, no scope', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-refresh-token': 'REFRESH_TOKEN' }, - query: {}, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - it('should request for a new access token and fail if no access token returned', () => { - const spyRefresh = jest - .spyOn(refresh, 'requestNewAccessToken') - .mockImplementation((_x, _y, _z, callbackFunc) => { - const cb = callbackFunc as Function; - cb(undefined, undefined, undefined, {}); - }); - - googleAuthProvider.refresh(mockRequest, mockResponse); - expect(spyRefresh).toBeCalledTimes(1); - expect(spyRefresh).toBeCalledWith( - 'google', - 'REFRESH_TOKEN', - {}, - expect.any(Function), - ); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith( - 'Failed to refresh access token', - ); - }); - - it('should request for a new access token and return 401 if any error', () => { - const spyRefresh = jest - .spyOn(refresh, 'requestNewAccessToken') - .mockImplementation((_x, _y, _z, callbackFunc) => { - const cb = callbackFunc as Function; - cb({ error: 'ERROR' }, undefined, undefined, {}); - }); - - googleAuthProvider.refresh(mockRequest, mockResponse); - expect(spyRefresh).toBeCalledTimes(1); - expect(spyRefresh).toBeCalledWith( - 'google', - 'REFRESH_TOKEN', - {}, - expect.any(Function), - ); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith( - 'Failed to refresh access token', - ); - }); - - it('should fetch and return a new access token', () => { - const spyRefresh = jest - .spyOn(refresh, 'requestNewAccessToken') - .mockImplementation((_x, _y, _z, callbackFunc) => { - const cb = callbackFunc as Function; - cb(undefined, 'ACCESS_TOKEN', undefined, { - expires_in: 'EXPIRES_IN', - id_token: 'ID_TOKEN', - }); - }); - - googleAuthProvider.refresh(mockRequest, mockResponse); - expect(spyRefresh).toBeCalledTimes(1); - expect(spyRefresh).toBeCalledWith( - 'google', - 'REFRESH_TOKEN', - {}, - expect.any(Function), - ); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith({ - accessToken: 'ACCESS_TOKEN', - idToken: 'ID_TOKEN', - expiresInSeconds: 'EXPIRES_IN', - scope: undefined, - }); - }); - }); - - describe('refresh token cookie and scope', () => { - const mockRequest = ({ - header: () => 'XMLHttpRequest', - cookies: { 'google-refresh-token': 'REFRESH_TOKEN' }, - query: { - scope: 'a,b', - }, - } as unknown) as express.Request; - - const googleAuthProvider = new GoogleAuthProvider( - googleAuthProviderConfig, - ); - - it('should fetch and return a new access token with scopes', () => { - const spyRefresh = jest - .spyOn(refresh, 'requestNewAccessToken') - .mockImplementation((_x, _y, _z, callbackFunc) => { - const cb = callbackFunc as Function; - cb(undefined, 'ACCESS_TOKEN', undefined, { - expires_in: 'EXPIRES_IN', - id_token: 'ID_TOKEN', - scope: 'a,b', - }); - }); - - googleAuthProvider.refresh(mockRequest, mockResponse); - expect(spyRefresh).toBeCalledTimes(1); - expect(spyRefresh).toBeCalledWith( - 'google', - 'REFRESH_TOKEN', - { scope: 'a,b' }, - expect.any(Function), - ); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith({ - accessToken: 'ACCESS_TOKEN', - idToken: 'ID_TOKEN', - expiresInSeconds: 'EXPIRES_IN', - scope: 'a,b', - }); - }); - - it('ensures x-requested-with header', () => { - const mockHeaderRequest = ({ - header: () => 'TEST', - } as unknown) as express.Request; - - googleAuthProvider.refresh(mockHeaderRequest, mockResponse); - expect(mockResponse.send).toBeCalledTimes(1); - expect(mockResponse.send).toBeCalledWith( - 'Invalid X-Requested-With header', - ); - expect(mockResponse.status).toBeCalledTimes(1); - expect(mockResponse.status).toBeCalledWith(401); - }); - }); - }); -}); diff --git a/plugins/auth-backend/src/providers/google/provider.ts b/plugins/auth-backend/src/providers/google/provider.ts index cb080e2fd3..90d33652a5 100644 --- a/plugins/auth-backend/src/providers/google/provider.ts +++ b/plugins/auth-backend/src/providers/google/provider.ts @@ -14,168 +14,29 @@ * limitations under the License. */ -import passport from 'passport'; -import express, { CookieOptions } from 'express'; -import crypto from 'crypto'; +import express from 'express'; import { Strategy as GoogleStrategy } from 'passport-google-oauth20'; -import refresh from 'passport-oauth2-refresh'; import { - AuthProvider, - AuthProviderRouteHandlers, + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, +} from '../PassportStrategyHelper'; +import { + OAuthProviderHandlers, + AuthInfoBase, + AuthInfoPrivate, + RedirectInfo, AuthProviderConfig, -} from './../types'; -import { postMessageResponse, ensuresXRequestedWith } from './../utils'; -import { InputError } from '@backstage/backend-common'; +} from '../types'; -export const THOUSAND_DAYS_MS = 1000 * 24 * 60 * 60 * 1000; -export const TEN_MINUTES_MS = 600 * 1000; -export class GoogleAuthProvider - implements AuthProvider, AuthProviderRouteHandlers { +export class GoogleAuthProvider implements OAuthProviderHandlers { private readonly providerConfig: AuthProviderConfig; + private readonly _strategy: GoogleStrategy; + constructor(providerConfig: AuthProviderConfig) { this.providerConfig = providerConfig; - } - - start( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ) { - const nonce = crypto.randomBytes(16).toString('base64'); - - const options: CookieOptions = { - maxAge: TEN_MINUTES_MS, - secure: false, - sameSite: 'none', - domain: 'localhost', - path: `/auth/${this.providerConfig.provider}/handler`, - httpOnly: true, - }; - - res.cookie(`${this.providerConfig.provider}-nonce`, nonce, options); - - const scope = req.query.scope?.toString() ?? ''; - if (!scope) { - throw new InputError('missing scope parameter'); - } - return passport.authenticate('google', { - scope, - accessType: 'offline', - prompt: 'consent', - state: nonce, - })(req, res, next); - } - - frameHandler( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ) { - const cookieNonce = req.cookies[`${this.providerConfig.provider}-nonce`]; - const stateNonce = req.query.state; - - if (!cookieNonce || !stateNonce) { - return res.status(401).send('Missing nonce'); - } - - if (cookieNonce !== stateNonce) { - return res.status(401).send('Invalid nonce'); - } - - return passport.authenticate('google', (err, user) => { - if (err) { - return postMessageResponse(res, { - type: 'auth-result', - error: new Error(`Google auth failed, ${err}`), - }); - } - - const { refreshToken } = user; - - if (!refreshToken) { - return postMessageResponse(res, { - type: 'auth-result', - error: new Error('Missing refresh token'), - }); - } - - delete user.refreshToken; - - const options: CookieOptions = { - maxAge: THOUSAND_DAYS_MS, - secure: false, - sameSite: 'none', - domain: 'localhost', - path: `/auth/${this.providerConfig.provider}`, - httpOnly: true, - }; - - res.cookie( - `${this.providerConfig.provider}-refresh-token`, - refreshToken, - options, - ); - return postMessageResponse(res, { - type: 'auth-result', - payload: user, - }); - })(req, res, next); - } - - async logout(req: express.Request, res: express.Response) { - if (!ensuresXRequestedWith(req)) { - return res.status(401).send('Invalid X-Requested-With header'); - } - - const options: CookieOptions = { - maxAge: 0, - secure: false, - sameSite: 'none', - domain: 'localhost', - path: `/auth/${this.providerConfig.provider}`, - httpOnly: true, - }; - - res.cookie(`${this.providerConfig.provider}-refresh-token`, '', options); - return res.send('logout!'); - } - - async refresh(req: express.Request, res: express.Response) { - if (!ensuresXRequestedWith(req)) { - return res.status(401).send('Invalid X-Requested-With header'); - } - - const refreshToken = - req.cookies[`${this.providerConfig.provider}-refresh-token`]; - - if (!refreshToken) { - return res.status(401).send('Missing session cookie'); - } - - const scope = req.query.scope?.toString() ?? ''; - const refreshTokenRequestParams = scope ? { scope } : {}; - - return refresh.requestNewAccessToken( - this.providerConfig.provider, - refreshToken, - refreshTokenRequestParams, - (err, accessToken, _refreshToken, params) => { - if (err || !accessToken) { - return res.status(401).send('Failed to refresh access token'); - } - return res.send({ - accessToken, - idToken: params.id_token, - expiresInSeconds: params.expires_in, - scope: params.scope, - }); - }, - ); - } - - strategy(): passport.Strategy { // TODO: throw error if env variables not set? - return new GoogleStrategy( + this._strategy = new GoogleStrategy( { ...this.providerConfig.options }, ( accessToken: any, @@ -184,15 +45,45 @@ export class GoogleAuthProvider profile: any, done: any, ) => { - done(undefined, { - profile, - idToken: params.id_token, - accessToken, - refreshToken, - scope: params.scope, - expiresInSeconds: params.expires_in, - }); + done( + undefined, + { + profile, + idToken: params.id_token, + accessToken, + scope: params.scope, + expiresInSeconds: params.expires_in, + }, + { + refreshToken, + }, + ); }, ); } + + async start(req: express.Request, options: any): Promise { + return await executeRedirectStrategy(req, this._strategy, options); + } + + async handler( + req: express.Request, + ): Promise<{ user: AuthInfoBase; info: AuthInfoPrivate }> { + return await executeFrameHandlerStrategy(req, this._strategy); + } + + async refresh(refreshToken: string, scope: string): Promise { + const { accessToken, params } = await executeRefreshTokenStrategy( + this._strategy, + refreshToken, + scope, + ); + + return { + accessToken, + idToken: params.id_token, + expiresInSeconds: params.expires_in, + scope: params.scope, + }; + } } diff --git a/plugins/auth-backend/src/providers/index.test.ts b/plugins/auth-backend/src/providers/index.test.ts index e42cd32edc..b3e2f19771 100644 --- a/plugins/auth-backend/src/providers/index.test.ts +++ b/plugins/auth-backend/src/providers/index.test.ts @@ -14,90 +14,8 @@ * limitations under the License. */ -import passport from 'passport'; -import express from 'express'; -import { makeProvider, defaultRouter } from '.'; -import { - AuthProvider, - AuthProviderRouteHandlers, - AuthProviderConfig, -} from './types'; -import * as passportGoogleOAuth20 from 'passport-google-oauth20'; -import { ProviderFactories } from './factories'; - -class MyAuthProvider implements AuthProvider, AuthProviderRouteHandlers { - private readonly providerConfig: AuthProviderConfig; - constructor(providerConfig: AuthProviderConfig) { - this.providerConfig = providerConfig; - } - - strategy(): passport.Strategy { - return new passportGoogleOAuth20.Strategy( - this.providerConfig.options, - () => {}, - ); - } - async start(_: express.Request, res: express.Response): Promise { - res.send('start'); - } - async frameHandler(_: express.Request, res: express.Response): Promise { - res.send('frameHandler'); - } - async logout(_: express.Request, res: express.Response): Promise { - res.send('logout'); - } -} - -class MyAuthProviderWithRefresh extends MyAuthProvider { - async refresh(_: express.Request, res: express.Response): Promise { - res.send('logout'); - } -} - -const providerConfig = { - provider: 'a', - options: { - clientID: 'somevalue', - }, -}; - -const providerConfigInvalid = { - provider: 'b', - options: { - clientID: 'somevalue', - }, -}; - -describe('makeProvider', () => { - it('makes a provider for Myauthprovider', () => { - jest - .spyOn(ProviderFactories, 'getProviderFactory') - .mockReturnValueOnce(MyAuthProvider); - const provider = makeProvider(providerConfig); - expect(provider.providerId).toEqual('a'); - expect(provider.strategy).toBeDefined(); - expect(provider.providerRouter).toBeDefined(); - }); - - it('throws an error when provider implementation does not exist', () => { - expect(() => { - makeProvider(providerConfigInvalid); - }).toThrow('Provider Implementation missing for : b auth provider'); - }); -}); - -describe('defaultRouter', () => { - it('make router for auth provider without refresh', () => { - expect( - defaultRouter(new MyAuthProvider({ provider: 'a', options: {} })), - ).toBeDefined(); - }); - - it('make router for auth provider with refresh', () => { - expect( - defaultRouter( - new MyAuthProviderWithRefresh({ provider: 'b', options: {} }), - ), - ).toBeDefined(); +describe('test', () => { + it('unbreaks the test runner', () => { + expect(true).toBeTruthy(); }); }); diff --git a/plugins/auth-backend/src/providers/index.ts b/plugins/auth-backend/src/providers/index.ts index 1b33391c27..59dd116fa6 100644 --- a/plugins/auth-backend/src/providers/index.ts +++ b/plugins/auth-backend/src/providers/index.ts @@ -17,6 +17,7 @@ import Router from 'express-promise-router'; import { AuthProviderRouteHandlers, AuthProviderConfig } from './types'; import { ProviderFactories } from './factories'; +import { OAuthProvider } from './OAuthProvider'; export const defaultRouter = (provider: AuthProviderRouteHandlers) => { const router = Router(); @@ -33,7 +34,8 @@ export const makeProvider = (config: AuthProviderConfig) => { const providerId = config.provider; const ProviderImpl = ProviderFactories.getProviderFactory(providerId); const providerInstance = new ProviderImpl(config); - const strategy = providerInstance.strategy(); - const providerRouter = defaultRouter(providerInstance); - return { providerId, strategy, providerRouter }; + + const oauthProvider = new OAuthProvider(providerInstance, providerId); + const providerRouter = defaultRouter(oauthProvider); + return { providerId, providerRouter }; }; diff --git a/plugins/auth-backend/src/providers/types.ts b/plugins/auth-backend/src/providers/types.ts index 36941c6850..dcbe6ad1de 100644 --- a/plugins/auth-backend/src/providers/types.ts +++ b/plugins/auth-backend/src/providers/types.ts @@ -22,32 +22,18 @@ export type AuthProviderConfig = { options: any; }; -export interface AuthProvider { - strategy(): passport.Strategy; - router?(): express.Router; +export interface OAuthProviderHandlers { + start(req: express.Request, options: any): Promise; + handler(req: express.Request): Promise; + refresh(refreshToken: string, scope: string): Promise; + logout?(): Promise; } export interface AuthProviderRouteHandlers { - start( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ): Promise; - frameHandler( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ): Promise; - refresh?( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ): Promise; - logout( - req: express.Request, - res: express.Response, - next: express.NextFunction, - ): Promise; + start(req: express.Request, res: express.Response): Promise; + frameHandler(req: express.Request, res: express.Response): Promise; + refresh?(req: express.Request, res: express.Response): Promise; + logout(req: express.Request, res: express.Response): Promise; } export type AuthProviderFactories = { @@ -55,7 +41,7 @@ export type AuthProviderFactories = { }; export type AuthProviderFactory = { - new (providerConfig: any): AuthProvider & AuthProviderRouteHandlers; + new (providerConfig: any): OAuthProviderHandlers; }; export type AuthInfoBase = { @@ -69,7 +55,7 @@ export type AuthInfoWithProfile = AuthInfoBase & { profile: passport.Profile; }; -export type AuthInfoPrivate = AuthInfoWithProfile & { +export type AuthInfoPrivate = { refreshToken: string; }; @@ -82,3 +68,13 @@ export type AuthResponse = type: 'auth-result'; error: Error; }; + +export type RedirectInfo = { + url: string; + status?: number; +}; + +export type RefreshTokenResponse = { + accessToken: string; + params: any; +}; diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index a2e3b3e1db..49487d551a 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -16,10 +16,7 @@ import express from 'express'; import Router from 'express-promise-router'; -import passport from 'passport'; import cookieParser from 'cookie-parser'; -import refresh from 'passport-oauth2-refresh'; -import OAuth2Strategy from 'passport-oauth2'; import { Logger } from 'winston'; import { providers } from './../providers/config'; import { makeProvider } from '../providers'; @@ -33,38 +30,14 @@ export async function createRouter( ): Promise { const router = Router(); const logger = options.logger.child({ plugin: 'auth' }); - const providerRouters: { [key: string]: express.Router } = {}; + + router.use(cookieParser()); // configure all the providers for (const providerConfig of providers) { - const { providerId, strategy, providerRouter } = makeProvider( - providerConfig, - ); - logger.info(`Configuring provider: ${providerId}`); - passport.use(strategy); - if (strategy instanceof OAuth2Strategy) { - refresh.use(strategy); - } - providerRouters[providerId] = providerRouter; - } - - passport.serializeUser((user, done) => { - done(null, user); - }); - - passport.deserializeUser((user, done) => { - done(null, user); - }); - - router.use(passport.initialize()); - router.use(passport.session()); - router.use(cookieParser()); - - for (const providerId in providerRouters) { - if (providerRouters.hasOwnProperty(providerId)) { - const providerRouter = providerRouters[providerId]; - router.use(`/${providerId}`, providerRouter); - } + const { providerId, providerRouter } = makeProvider(providerConfig); + logger.info(`Configuring provider, ${providerId}`); + router.use(`/${providerId}`, providerRouter); } return router; diff --git a/plugins/auth-backend/src/setupTests.ts b/plugins/auth-backend/src/setupTests.ts index 3fa7cb04b4..a3b2b7e123 100644 --- a/plugins/auth-backend/src/setupTests.ts +++ b/plugins/auth-backend/src/setupTests.ts @@ -15,3 +15,4 @@ */ require('jest-fetch-mock').enableMocks(); +export {}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f32ffffb6c..5ec37295ed 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -16,8 +16,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.6", - "@types/node-fetch": "^2.5.7", - "@types/supertest": "^2.0.8", + "@backstage/catalog-model": "^0.1.1-alpha.6", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -38,6 +37,8 @@ "devDependencies": { "@backstage/cli": "^0.1.1-alpha.6", "@types/lodash": "^4.14.151", + "@types/node-fetch": "^2.5.7", + "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.28.2", "jest-fetch-mock": "^3.0.3", diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 9d0d8fcaf7..972410a639 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -14,21 +14,21 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { Database } from '../database'; -import { DescriptorEnvelope } from '../ingestion/types'; import { EntitiesCatalog, EntityFilters } from './types'; export class DatabaseEntitiesCatalog implements EntitiesCatalog { constructor(private readonly database: Database) {} - async entities(filters?: EntityFilters): Promise { + async entities(filters?: EntityFilters): Promise { const items = await this.database.transaction(tx => this.database.entities(tx, filters), ); return items.map(i => i.entity); } - async entityByUid(uid: string): Promise { + async entityByUid(uid: string): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [{ key: 'uid', values: [uid] }]), ); @@ -40,7 +40,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const matches = await this.database.transaction(tx => this.database.entities(tx, [ { key: 'kind', values: [kind] }, diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts index 0181b7fc28..56a3b3828f 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts @@ -13,13 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; +import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import knex from 'knex'; import path from 'path'; - import { Database } from '../database'; -import { ReaderOutput } from '../ingestion/types'; -import { getVoidLogger } from '@backstage/backend-common'; +import { IngestionModel } from '../ingestion/types'; +import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; + +class MockIngestionModel implements IngestionModel { + readLocation = jest.fn(async (type: string, target: string) => { + if (type !== 'valid_type') { + throw new Error(`Unknown location type ${type}`); + } + if (target === 'valid_target') { + return [{ type: 'data', data: {} as Entity } as const]; + } + throw new Error( + `Can't read location at ${target} with error: Something is broken`, + ); + }); +} describe('DatabaseLocationsCatalog', () => { const database = knex({ @@ -32,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => { }); let db: Database; let catalog: DatabaseLocationsCatalog; - - const mockLocationReader = { - read: async (type: string, target: string): Promise => { - if (type !== 'valid_type') { - throw new Error(`Unknown location type ${type}`); - } - if (target === 'valid_target') { - return Promise.resolve([{ type: 'data', data: {} }]); - } - throw new Error( - `Can't read location at ${target} with error: Something is broken`, - ); - }, - }; + let ingestionModel: IngestionModel; beforeEach(async () => { await database.migrate.latest({ @@ -53,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => { loadExtensions: ['.ts'], }); db = new Database(database, getVoidLogger()); - catalog = new DatabaseLocationsCatalog(db, mockLocationReader); + ingestionModel = new MockIngestionModel(); + catalog = new DatabaseLocationsCatalog(db, ingestionModel); }); it('resolves to location with id', async () => { diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 6e3c9ae306..b13d70de86 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -15,17 +15,25 @@ */ import { Database } from '../database'; +import { IngestionModel } from '../ingestion/types'; import { AddLocation, Location, LocationsCatalog } from './types'; -import { LocationReader } from '../ingestion'; export class DatabaseLocationsCatalog implements LocationsCatalog { constructor( private readonly database: Database, - private readonly reader: LocationReader, + private readonly ingestionModel: IngestionModel, ) {} async addLocation(location: AddLocation): Promise { - const outputs = await this.reader.read(location.type, location.target); + const outputs = await this.ingestionModel.readLocation( + location.type, + location.target, + ); + if (!outputs) { + throw new Error( + `Unknown location type ${location.type} ${location.target}`, + ); + } outputs.forEach(output => { if (output.type === 'error') { throw new Error( diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts index 371cdf1f76..64ac5f57d5 100644 --- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts @@ -15,23 +15,23 @@ */ import { NotFoundError } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import lodash from 'lodash'; -import { DescriptorEnvelope } from '../ingestion'; import { EntitiesCatalog } from './types'; export class StaticEntitiesCatalog implements EntitiesCatalog { - private _entities: DescriptorEnvelope[]; + private _entities: Entity[]; - constructor(entities: DescriptorEnvelope[]) { + constructor(entities: Entity[]) { this._entities = entities; } - async entities(): Promise { + async entities(): Promise { return lodash.cloneDeep(this._entities); } - async entityByUid(uid: string): Promise { - const item = this._entities.find(e => uid === e.metadata?.uid); + async entityByUid(uid: string): Promise { + const item = this._entities.find(e => uid === e.metadata.uid); if (!item) { throw new NotFoundError('Entity cannot be found'); } @@ -42,12 +42,12 @@ export class StaticEntitiesCatalog implements EntitiesCatalog { kind: string, name: string, namespace: string | undefined, - ): Promise { + ): Promise { const item = this._entities.find( e => kind === e.kind && - name === e.metadata?.name && - namespace === e.metadata?.namespace, + name === e.metadata.name && + namespace === e.metadata.namespace, ); if (!item) { throw new NotFoundError('Entity cannot be found'); diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index d5627ea86a..2f8967cd1b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; // // Entities @@ -28,13 +28,13 @@ export type EntityFilter = { export type EntityFilters = EntityFilter[]; export type EntitiesCatalog = { - entities(filters?: EntityFilters): Promise; - entityByUid(uid: string): Promise; + entities(filters?: EntityFilters): Promise; + entityByUid(uid: string): Promise; entityByName( kind: string, namespace: string | undefined, name: string, - ): Promise; + ): Promise; }; // diff --git a/plugins/catalog-backend/src/database/Database.test.ts b/plugins/catalog-backend/src/database/Database.test.ts index a82385aae4..bb1a66ac7c 100644 --- a/plugins/catalog-backend/src/database/Database.test.ts +++ b/plugins/catalog-backend/src/database/Database.test.ts @@ -19,9 +19,9 @@ import { getVoidLogger, NotFoundError, } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import Knex from 'knex'; import path from 'path'; -import { DescriptorEnvelope } from '../ingestion'; import { Database } from './Database'; import { AddDatabaseLocation, @@ -128,7 +128,7 @@ describe('Database', () => { catalog.addEntity(tx, entityRequest), ); expect(added).toStrictEqual(entityResponse); - expect(added.entity.metadata!.generation).toBe(1); + expect(added.entity.metadata.generation).toBe(1); }); it('rejects adding the same-named entity twice', async () => { @@ -141,9 +141,9 @@ describe('Database', () => { it('accepts adding the same-named entity twice if on different namespaces', async () => { const catalog = new Database(database, getVoidLogger()); - entityRequest.entity.metadata!.namespace = 'namespace1'; + entityRequest.entity.metadata.namespace = 'namespace1'; await catalog.transaction(tx => catalog.addEntity(tx, entityRequest)); - entityRequest.entity.metadata!.namespace = 'namespace2'; + entityRequest.entity.metadata.namespace = 'namespace2'; await expect( catalog.transaction(tx => catalog.addEntity(tx, entityRequest)), ).resolves.toBeDefined(); @@ -161,17 +161,15 @@ describe('Database', () => { ); expect(updated.entity.apiVersion).toEqual(added.entity.apiVersion); expect(updated.entity.kind).toEqual(added.entity.kind); - expect(updated.entity.metadata!.etag).not.toEqual( - added.entity.metadata!.etag, + expect(updated.entity.metadata.etag).not.toEqual( + added.entity.metadata.etag, ); - expect(updated.entity.metadata!.generation).toEqual( - added.entity.metadata!.generation, + expect(updated.entity.metadata.generation).toEqual( + added.entity.metadata.generation, ); - expect(updated.entity.metadata!.name).toEqual( - added.entity.metadata!.name, - ); - expect(updated.entity.metadata!.namespace).toEqual( - added.entity.metadata!.namespace, + expect(updated.entity.metadata.name).toEqual(added.entity.metadata.name); + expect(updated.entity.metadata.namespace).toEqual( + added.entity.metadata.namespace, ); }); @@ -180,11 +178,11 @@ describe('Database', () => { const added = await catalog.transaction(tx => catalog.addEntity(tx, entityRequest), ); - added.entity.metadata!.name! = 'new!'; + added.entity.metadata.name! = 'new!'; const updated = await catalog.transaction(tx => catalog.updateEntity(tx, { entity: added.entity }), ); - expect(updated.entity.metadata!.name).toEqual('new!'); + expect(updated.entity.metadata.name).toEqual('new!'); }); it('can update fields if kind, name, and namespace match', async () => { @@ -193,8 +191,8 @@ describe('Database', () => { catalog.addEntity(tx, entityRequest), ); added.entity.apiVersion = 'something.new'; - delete added.entity.metadata!.uid; - delete added.entity.metadata!.generation; + delete added.entity.metadata.uid; + delete added.entity.metadata.generation; const updated = await catalog.transaction(tx => catalog.updateEntity(tx, { entity: added.entity }), ); @@ -207,9 +205,9 @@ describe('Database', () => { catalog.addEntity(tx, entityRequest), ); added.entity.apiVersion = 'something.new'; - delete added.entity.metadata!.uid; - delete added.entity.metadata!.generation; - added.entity.metadata!.namespace = 'something.wrong'; + delete added.entity.metadata.uid; + delete added.entity.metadata.generation; + added.entity.metadata.namespace = 'something.wrong'; await expect( catalog.transaction(tx => catalog.updateEntity(tx, { entity: added.entity }), @@ -222,7 +220,7 @@ describe('Database', () => { const added = await catalog.transaction(tx => catalog.addEntity(tx, entityRequest), ); - added.entity.metadata!.etag = 'garbage'; + added.entity.metadata.etag = 'garbage'; await expect( catalog.transaction(tx => catalog.updateEntity(tx, { entity: added.entity }), @@ -235,7 +233,7 @@ describe('Database', () => { const added = await catalog.transaction(tx => catalog.addEntity(tx, entityRequest), ); - added.entity.metadata!.generation! += 100; + added.entity.metadata.generation! += 100; await expect( catalog.transaction(tx => catalog.updateEntity(tx, { entity: added.entity }), @@ -247,10 +245,15 @@ describe('Database', () => { describe('entities', () => { it('can get all entities with empty filters list', async () => { const catalog = new Database(database, getVoidLogger()); - const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' }; - const e2: DescriptorEnvelope = { + const e1: Entity = { apiVersion: 'a', - kind: 'b', + kind: 'k1', + metadata: { name: 'n' }, + }; + const e2: Entity = { + apiVersion: 'c', + kind: 'k2', + metadata: { name: 'n' }, spec: { c: null }, }; await catalog.transaction(async tx => { @@ -263,24 +266,32 @@ describe('Database', () => { expect(result.length).toEqual(2); expect(result).toEqual( expect.arrayContaining([ - { locationId: undefined, entity: expect.objectContaining(e1) }, - { locationId: undefined, entity: expect.objectContaining(e2) }, + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'k1' }), + }, + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'k2' }), + }, ]), ); }); it('can get all specific entities for matching filters (naive case)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ - { apiVersion: 'a', kind: 'b' }, + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } }, { apiVersion: 'a', - kind: 'b', + kind: 'k2', + metadata: { name: 'n' }, spec: { c: 'some' }, }, { apiVersion: 'a', - kind: 'b', + kind: 'k3', + metadata: { name: 'n' }, spec: { c: null }, }, ]; @@ -294,27 +305,32 @@ describe('Database', () => { await expect( catalog.transaction(async tx => catalog.entities(tx, [ - { key: 'kind', values: ['b'] }, + { key: 'kind', values: ['k2'] }, { key: 'spec.c', values: ['some'] }, ]), ), ).resolves.toEqual([ - { locationId: undefined, entity: expect.objectContaining(entities[1]) }, + { + locationId: undefined, + entity: expect.objectContaining({ kind: 'k2' }), + }, ]); }); it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => { const catalog = new Database(database, getVoidLogger()); - const entities: DescriptorEnvelope[] = [ - { apiVersion: 'a', kind: 'b' }, + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'k1', metadata: { name: 'n' } }, { apiVersion: 'a', - kind: 'b', + kind: 'k2', + metadata: { name: 'n' }, spec: { c: 'some' }, }, { apiVersion: 'a', - kind: 'b', + kind: 'k3', + metadata: { name: 'n' }, spec: { c: null }, }, ]; @@ -327,7 +343,7 @@ describe('Database', () => { const rows = await catalog.transaction(async tx => catalog.entities(tx, [ - { key: 'kind', values: ['b'] }, + { key: 'apiVersion', values: ['a'] }, { key: 'spec.c', values: [null, 'some'] }, ]), ); @@ -337,15 +353,15 @@ describe('Database', () => { expect.arrayContaining([ { locationId: undefined, - entity: expect.objectContaining(entities[0]), + entity: expect.objectContaining({ kind: 'k1' }), }, { locationId: undefined, - entity: expect.objectContaining(entities[1]), + entity: expect.objectContaining({ kind: 'k2' }), }, { locationId: undefined, - entity: expect.objectContaining(entities[2]), + entity: expect.objectContaining({ kind: 'k3' }), }, ]), ); diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index 614f7d09bf..26deb3362e 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -19,12 +19,12 @@ import { InputError, NotFoundError, } from '@backstage/backend-common'; +import { Entity, EntityMeta } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; import { EntityFilters } from '../catalog'; -import { DescriptorEnvelope, EntityMeta } from '../ingestion'; import { buildEntitySearch } from './search'; import { AddDatabaseLocation, @@ -46,17 +46,11 @@ function getStrippedMetadata(metadata: EntityMeta): EntityMeta { return output; } -function serializeMetadata(metadata: EntityMeta | undefined): string | null { - if (!metadata) { - return null; - } - +function serializeMetadata(metadata: EntityMeta): string { return JSON.stringify(getStrippedMetadata(metadata)); } -function serializeSpec( - spec: DescriptorEnvelope['spec'], -): DbEntitiesRow['spec'] { +function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] { if (!spec) { return null; } @@ -66,38 +60,34 @@ function serializeSpec( function toEntityRow( locationId: string | undefined, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesRow { return { - id: entity.metadata!.uid!, + id: entity.metadata.uid!, location_id: locationId || null, - etag: entity.metadata!.etag!, - generation: entity.metadata!.generation!, + etag: entity.metadata.etag!, + generation: entity.metadata.generation!, api_version: entity.apiVersion, kind: entity.kind, - name: entity.metadata!.name || null, - namespace: entity.metadata!.namespace || null, + name: entity.metadata.name || null, + namespace: entity.metadata.namespace || null, metadata: serializeMetadata(entity.metadata), spec: serializeSpec(entity.spec), }; } function toEntityResponse(row: DbEntitiesRow): DbEntityResponse { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: row.api_version, kind: row.kind, metadata: { + ...(JSON.parse(row.metadata) as Entity['metadata']), uid: row.id, etag: row.etag, generation: Number(row.generation), // cast because of sqlite }, }; - if (row.metadata) { - const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata']; - entity.metadata = { ...entity.metadata, ...metadata }; - } - if (row.spec) { const spec = JSON.parse(row.spec); entity.spec = spec; @@ -178,11 +168,11 @@ export class Database { tx: Knex.Transaction, request: DbEntityRequest, ): Promise { - if (request.entity.metadata?.uid !== undefined) { + if (request.entity.metadata.uid !== undefined) { throw new InputError('May not specify uid for new entities'); - } else if (request.entity.metadata?.etag !== undefined) { + } else if (request.entity.metadata.etag !== undefined) { throw new InputError('May not specify etag for new entities'); - } else if (request.entity.metadata?.generation !== undefined) { + } else if (request.entity.metadata.generation !== undefined) { throw new InputError('May not specify generation for new entities'); } @@ -289,9 +279,9 @@ export class Database { if (oldRow.metadata) { const oldMetadata = JSON.parse(oldRow.metadata) as EntityMeta; if (oldMetadata.annotations) { - newEntity.metadata!.annotations = { + newEntity.metadata.annotations = { ...oldMetadata.annotations, - ...newEntity.metadata!.annotations, + ...newEntity.metadata.annotations, }; } } @@ -422,7 +412,7 @@ export class Database { private async updateEntitiesSearch( tx: Knex.Transaction, entityId: string, - data: DescriptorEnvelope, + data: Entity, ): Promise { try { const entries = buildEntitySearch(entityId, data); diff --git a/plugins/catalog-backend/src/database/DatabaseManager.test.ts b/plugins/catalog-backend/src/database/DatabaseManager.test.ts index ca50f518cb..9a4297904a 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.test.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.test.ts @@ -15,13 +15,9 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; -import { - ComponentDescriptor, - DescriptorParser, - LocationReader, - ParserError, -} from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { Database } from './Database'; import { DatabaseManager } from './DatabaseManager'; import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types'; @@ -32,18 +28,18 @@ describe('DatabaseManager', () => { const db = ({ locations: jest.fn().mockResolvedValue([]), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(), + const reader: IngestionModel = { + readLocation: jest.fn(), }; - const parser: DescriptorParser = { - parse: jest.fn(), + const policy: EntityPolicy = { + enforce: jest.fn(), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).not.toHaveBeenCalled(); - expect(parser.parse).not.toHaveBeenCalled(); + expect(reader.readLocation).not.toHaveBeenCalled(); + expect(policy.enforce).not.toHaveBeenCalled(); }); it('can update a single location', async () => { @@ -52,7 +48,7 @@ describe('DatabaseManager', () => { type: 'some', target: 'thing', }; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, @@ -68,18 +64,20 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as Partial) as Database; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + enforce: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); - expect(reader.read).toHaveBeenCalledTimes(1); - expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing'); + expect(reader.readLocation).toHaveBeenCalledTimes(1); + expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing'); expect(db.addEntity).toHaveBeenCalledTimes(1); expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, { locationId: '123', @@ -108,21 +106,23 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => Promise.resolve(desc)), + const policy: EntityPolicy = { + enforce: jest.fn(() => Promise.resolve(desc)), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -158,23 +158,25 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const desc: ComponentDescriptor = { + const desc: Entity = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { name: 'c1' }, spec: { type: 'service' }, }; - const reader: LocationReader = { - read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])), + const reader: IngestionModel = { + readLocation: jest.fn(() => + Promise.resolve([{ type: 'data', data: desc }]), + ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), + const policy: EntityPolicy = { + enforce: jest.fn(() => + Promise.reject(new Error('parser error message')), ), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( @@ -211,19 +213,19 @@ describe('DatabaseManager', () => { addLocationUpdateLogEvent: jest.fn(), } as unknown) as Database; - const reader: LocationReader = { - read: jest.fn(() => + const reader: IngestionModel = { + readLocation: jest.fn(() => Promise.reject([{ type: 'error', error: new Error('test message') }]), ), }; - const parser: DescriptorParser = { - parse: jest.fn(() => - Promise.reject(new ParserError('parser error message', 'c1')), + const policy: EntityPolicy = { + enforce: jest.fn(() => + Promise.reject(new Error('parser error message')), ), }; await expect( - DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()), + DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()), ).resolves.toBeUndefined(); expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith( diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index 7f646971db..05b6eaef8d 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -14,16 +14,12 @@ * limitations under the License. */ +import { Entity, EntityPolicy } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import path from 'path'; import { Logger } from 'winston'; -import { - DescriptorEnvelope, - DescriptorParser, - LocationReader, - ParserError, -} from '../ingestion'; +import { IngestionModel } from '../ingestion/types'; import { Database } from './Database'; import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types'; @@ -67,8 +63,8 @@ export class DatabaseManager { public static async refreshLocations( database: Database, - reader: LocationReader, - parser: DescriptorParser, + ingestionModel: IngestionModel, + entityPolicy: EntityPolicy, logger: Logger, ): Promise { const locations = await database.locations(); @@ -78,7 +74,10 @@ export class DatabaseManager { `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`, ); - const readerOutput = await reader.read(location.type, location.target); + const readerOutput = await ingestionModel.readLocation( + location.type, + location.target, + ); for (const readerItem of readerOutput) { if (readerItem.type === 'error') { @@ -87,7 +86,7 @@ export class DatabaseManager { } try { - const entity = await parser.parse(readerItem.data); + const entity = await entityPolicy.enforce(readerItem.data); await DatabaseManager.refreshSingleEntity( database, location.id, @@ -97,18 +96,14 @@ export class DatabaseManager { await DatabaseManager.logUpdateSuccess( database, location.id, - entity.metadata!.name, + entity.metadata.name, ); } catch (error) { - let entityName; - if (error instanceof ParserError) { - entityName = error.entityName; - } await DatabaseManager.logUpdateFailure( database, location.id, error, - entityName, + readerItem.data.metadata.name, ); } } @@ -129,7 +124,7 @@ export class DatabaseManager { private static async refreshSingleEntity( database: Database, locationId: string, - entity: DescriptorEnvelope, + entity: Entity, logger: Logger, ): Promise { const { kind } = entity; @@ -163,10 +158,7 @@ export class DatabaseManager { }); } - private static entitiesAreEqual( - first: DescriptorEnvelope, - second: DescriptorEnvelope, - ) { + private static entitiesAreEqual(first: Entity, second: Entity) { const firstClone = lodash.cloneDeep(first); const secondClone = lodash.cloneDeep(second); diff --git a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts index f029871a70..2ff06c4046 100644 --- a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts +++ b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts @@ -76,7 +76,7 @@ export async function up(knex: Knex): Promise { .comment('The metadata.namespace field of the entity'); table .string('metadata') - .nullable() + .notNullable() .comment('The entire metadata JSON blob of the entity'); table .string('spec') diff --git a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts index b2e1dc0d32..6700f5748e 100644 --- a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts +++ b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts @@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise { return knex.schema.createTable('location_update_log', table => { table.uuid('id').primary(); table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable(); table.string('message'); table .uuid('location_id') diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts index 26b20d1846..8f011fb250 100644 --- a/plugins/catalog-backend/src/database/search.test.ts +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { buildEntitySearch, visitEntityPart } from './search'; import { DbEntitiesSearchRow } from './types'; @@ -99,24 +99,25 @@ describe('search', () => { describe('buildEntitySearch', () => { it('adds special keys even if missing', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', + metadata: { name: 'n' }, }; expect(buildEntitySearch('eid', input)).toEqual([ - { entity_id: 'eid', key: 'metadata.name', value: null }, + { entity_id: 'eid', key: 'metadata.name', value: 'n' }, { entity_id: 'eid', key: 'metadata.namespace', value: null }, { entity_id: 'eid', key: 'metadata.uid', value: null }, { entity_id: 'eid', key: 'apiVersion', value: 'a' }, { entity_id: 'eid', key: 'kind', value: 'b' }, - { entity_id: 'eid', key: 'name', value: null }, + { entity_id: 'eid', key: 'name', value: 'n' }, { entity_id: 'eid', key: 'namespace', value: null }, { entity_id: 'eid', key: 'uid', value: null }, ]); }); it('adds prefix-stripped versions', () => { - const input: DescriptorEnvelope = { + const input: Entity = { apiVersion: 'a', kind: 'b', metadata: { diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts index f54cfde7e6..87fc59185d 100644 --- a/plugins/catalog-backend/src/database/search.ts +++ b/plugins/catalog-backend/src/database/search.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DescriptorEnvelope } from '../ingestion'; +import { Entity } from '@backstage/catalog-model'; import { DbEntitiesSearchRow } from './types'; // Search entries that start with these prefixes, also get a shorthand without @@ -119,7 +119,7 @@ export function visitEntityPart( */ export function buildEntitySearch( entityId: string, - entity: DescriptorEnvelope, + entity: Entity, ): DbEntitiesSearchRow[] { // Start with some special keys that are always present because you want to // be able to easily search for null specifically @@ -127,17 +127,17 @@ export function buildEntitySearch( { entity_id: entityId, key: 'metadata.name', - value: toValue(entity.metadata?.name), + value: toValue(entity.metadata.name), }, { entity_id: entityId, key: 'metadata.namespace', - value: toValue(entity.metadata?.namespace), + value: toValue(entity.metadata.namespace), }, { entity_id: entityId, key: 'metadata.uid', - value: toValue(entity.metadata?.uid), + value: toValue(entity.metadata.uid), }, ]; diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 06e234545f..9b5a91d249 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import * as yup from 'yup'; -import { DescriptorEnvelope } from '../ingestion'; export type DbEntitiesRow = { id: string; @@ -26,18 +26,18 @@ export type DbEntitiesRow = { namespace: string | null; etag: string; generation: number; - metadata: string | null; + metadata: string; spec: string | null; }; export type DbEntityRequest = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntityResponse = { locationId?: string; - entity: DescriptorEnvelope; + entity: Entity; }; export type DbEntitiesSearchRow = { diff --git a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts deleted file mode 100644 index 86283df666..0000000000 --- a/plugins/catalog-backend/src/ingestion/DescriptorParsers.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { makeValidator } from '../validation'; -import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser'; -import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser'; -import { - DescriptorEnvelope, - DescriptorParser, - KindParser, - ParserError, -} from './types'; - -export class DescriptorParsers implements DescriptorParser { - static create(): DescriptorParser { - const validators = makeValidator(); - return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [ - new ComponentDescriptorV1beta1Parser(), - ]); - } - - constructor( - private readonly envelopeParser: DescriptorEnvelopeParser, - private readonly kindParsers: KindParser[], - ) {} - - async parse(descriptor: object): Promise { - const envelope = await this.envelopeParser.parse(descriptor); - for (const parser of this.kindParsers) { - const parsed = await parser.tryParse(envelope); - if (parsed) { - return parsed; - } - } - throw new ParserError( - `Unsupported object ${envelope.apiVersion}, ${envelope.kind}`, - envelope.metadata?.name, - ); - } -} diff --git a/plugins/catalog-backend/src/ingestion/IngestionModels.ts b/plugins/catalog-backend/src/ingestion/IngestionModels.ts new file mode 100644 index 0000000000..def6f1fa5c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/IngestionModels.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model'; +import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types'; +import { LocationReader, LocationReaders } from './source'; +import { IngestionModel } from './types'; +import { DescriptorParsers } from './descriptor'; + +export class IngestionModels implements IngestionModel { + private readonly reader: LocationReader; + private readonly parser: DescriptorParser; + private readonly entityPolicy: EntityPolicy; + + static default(): IngestionModel { + return new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); + } + + constructor( + reader: LocationReader, + parser: DescriptorParser, + entityPolicy: EntityPolicy, + ) { + this.reader = reader; + this.parser = parser; + this.entityPolicy = entityPolicy; + } + + async readLocation(type: string, target: string): Promise { + const buffer = await this.reader.tryRead(type, target); + if (!buffer) { + throw new Error(`No reader could handle location ${type} ${target}`); + } + + const items = await this.parser.tryParse(buffer); + if (!items) { + throw new Error(`No parser could handle location ${type} ${target}`); + } + + const result: ReaderOutput[] = []; + for (const item of items) { + if (item.type === 'error') { + result.push(item); + } else { + try { + const output = await this.entityPolicy.enforce(item.data); + result.push({ type: 'data', data: output }); + } catch (e) { + result.push({ type: 'error', error: e }); + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts deleted file mode 100644 index aa88d299e4..0000000000 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { FileLocationSource } from './sources/FileLocationSource'; -import { GitHubLocationSource } from './sources/GitHubLocationSource'; -import { LocationReader, LocationSource, ReaderOutput } from './types'; - -export class LocationReaders implements LocationReader { - static create(): LocationReader { - return new LocationReaders({ - file: new FileLocationSource(), - github: new GitHubLocationSource(), - }); - } - - constructor(private readonly sources: Record) {} - - async read(type: string, target: string): Promise { - const source = this.sources[type]; - if (!source) { - throw new Error(`Unknown location type ${type}`); - } - - return source.read(target); - } -} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts new file mode 100644 index 0000000000..ed05855109 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DescriptorParser, ReaderOutput } from './parsers/types'; +import { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; + +/** + * Parses raw descriptor data (e.g. from a file or stream) into entities. + */ +export class DescriptorParsers implements DescriptorParser { + private readonly parsers: DescriptorParser[]; + + static defaultParsers(): DescriptorParser[] { + return [new YamlDescriptorParser()]; + } + + constructor( + parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(), + ) { + this.parsers = parsers; + } + + async tryParse(data: Buffer): Promise { + for (const parser of this.parsers) { + const result = await parser.tryParse(data); + if (result) { + return result; + } + } + throw new Error(`Unsupported descriptor format`); + } +} diff --git a/plugins/auth-backend/src/index.test.ts b/plugins/catalog-backend/src/ingestion/descriptor/index.ts similarity index 82% rename from plugins/auth-backend/src/index.test.ts rename to plugins/catalog-backend/src/ingestion/descriptor/index.ts index b3e2f19771..1529c78afc 100644 --- a/plugins/auth-backend/src/index.test.ts +++ b/plugins/catalog-backend/src/ingestion/descriptor/index.ts @@ -14,8 +14,5 @@ * limitations under the License. */ -describe('test', () => { - it('unbreaks the test runner', () => { - expect(true).toBeTruthy(); - }); -}); +export { DescriptorParsers } from './DescriptorParsers'; +export { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts new file mode 100644 index 0000000000..3f2e9e3c5c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import yaml from 'yaml'; +import { DescriptorParser, ReaderOutput } from './types'; + +/** + * Parses descriptors on YAML format + */ +export class YamlDescriptorParser implements DescriptorParser { + async tryParse(data: Buffer): Promise { + // TODO(freben): Should perhaps first do format detection, so the parse + // failure can be emitted as a proper error instead of just as if we + // weren't handling the format at all. + let documents; + try { + documents = yaml.parseAllDocuments(data.toString('utf8')); + } catch (e) { + return undefined; + } + + const result: ReaderOutput[] = []; + + for (const document of documents) { + if (document.contents) { + if (document.errors?.length) { + result.push({ + type: 'error', + error: new Error(`Malformed YAML document, ${document.errors[0]}`), + }); + } else { + const json = document.toJSON(); + if (typeof json !== 'object' || Array.isArray(json)) { + result.push({ + type: 'error', + error: new Error(`Malformed descriptor, expected object at root`), + }); + } else { + result.push({ + type: 'data', + data: json as Entity, + }); + } + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts new file mode 100644 index 0000000000..baef5dc3df --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; + +export type ReaderOutput = + | { type: 'error'; error: Error } + | { type: 'data'; data: Entity }; + +/** + * Parses raw descriptor data (e.g. from a file) into entities. + */ +export type DescriptorParser = { + /** + * Try to parse some raw data into an entity. + * + * Note that this is only the low level operation of parsing the raw file + * format, e.g. reading JSON or YAML or similar and emitting as structured + * but unvalidated data. The actual validation is performed by EntityPolicy + * and KindParser. + * + * @param data Raw descriptor data + * @returns A list of raw unvalidated entities / errors, or undefined if the + * given data is not meant to be handled by this parser + * @throws An Error if the format was handled and found to not be properly + * formed + */ + tryParse(data: Buffer): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts deleted file mode 100644 index 7c96fb7cf5..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { makeValidator } from '../../validation'; -import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser'; - -describe('DescriptorEnvelopeParser', () => { - let data: any; - let parser: DescriptorEnvelopeParser; - - beforeEach(() => { - data = yaml.parse(` - apiVersion: backstage.io/v1beta1 - kind: Component - metadata: - uid: e01199ab-08cc-44c2-8e19-5c29ded82521 - etag: lsndfkjsndfkjnsdfkjnsd== - generation: 13 - name: my-component-yay - namespace: the-namespace - labels: - backstage.io/custom: ValueStuff - annotations: - example.com/bindings: are-secret - spec: - custom: stuff - `); - parser = new DescriptorEnvelopeParser(makeValidator()); - }); - - it('works for the happy path', async () => { - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects missing apiVersion', async () => { - delete data.apiVersion; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects wrong root type', async () => { - await expect(parser.parse(7)).rejects.toThrow(/object/); - }); - - it('rejects bad apiVersion', async () => { - data.apiVersion = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/apiVersion/); - }); - - it('rejects missing kind', async () => { - delete data.kind; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('rejects bad kind', async () => { - data.kind = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/kind/); - }); - - it('accepts missing metadata', async () => { - delete data.medatata; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object metadata', async () => { - data.metadata = 7; - await expect(parser.parse(data)).rejects.toThrow(/metadata/); - }); - - it('accepts missing uid', async () => { - delete data.metadata.uid; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad uid', async () => { - data.metadata.uid = 7; - await expect(parser.parse(data)).rejects.toThrow(/uid/); - }); - - it('accepts missing etag', async () => { - delete data.metadata.etag; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad etag', async () => { - data.metadata.etag = 7; - await expect(parser.parse(data)).rejects.toThrow(/etag/); - }); - - it('accepts missing generation', async () => { - delete data.metadata.generation; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects bad generation', async () => { - data.metadata.generation = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/generation/); - }); - - it('accepts missing spec', async () => { - delete data.spec; - await expect(parser.parse(data)).resolves.toBe(data); - }); - - it('rejects non-object spec', async () => { - data.spec = 7; - await expect(parser.parse(data)).rejects.toThrow(/spec/); - }); - - it('rejects bad name', async () => { - data.metadata.name = 7; - await expect(parser.parse(data)).rejects.toThrow(/name/); - }); - - it('rejects bad namespace', async () => { - data.metadata.namespace = 7; - await expect(parser.parse(data)).rejects.toThrow(/namespace/); - }); - - it('rejects bad label key', async () => { - data.metadata.labels['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/label.*key/i); - }); - - it('rejects bad label value', async () => { - data.metadata.labels.a = 'a#b'; - await expect(parser.parse(data)).rejects.toThrow(/label.*value/i); - }); - - it('rejects bad annotation key', async () => { - data.metadata.annotations['a#b'] = 'value'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i); - }); - - it('rejects bad annotation value', async () => { - data.metadata.annotations.a = []; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i); - }); - - it('rejects unknown root keys', async () => { - data.spec2 = {}; - await expect(parser.parse(data)).rejects.toThrow(/spec2/i); - }); - - it('rejects reserved keys in the spec root', async () => { - data.spec.apiVersion = 'a/b'; - await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i); - }); - - it('rejects reserved keys in labels', async () => { - data.metadata.labels.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i); - }); - - it('rejects reserved keys in annotations', async () => { - data.metadata.annotations.apiVersion = 'a'; - await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts b/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts deleted file mode 100644 index 1f21e2df9e..0000000000 --- a/plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as yup from 'yup'; -import { Validators } from '../../validation'; -import { DescriptorEnvelope } from '../types'; - -/** - * Parses some raw structured data as a descriptor envelope - */ -export class DescriptorEnvelopeParser { - private schema: yup.Schema; - - constructor(validators: Validators) { - const apiVersionSchema = yup - .string() - .required() - .test( - 'apiVersion', - 'The apiVersion is not formatted according to schema', - validators.isValidApiVersion, - ); - - const kindSchema = yup - .string() - .required() - .test( - 'kind', - 'The kind is not formatted according to schema', - validators.isValidKind, - ); - - const uidSchema = yup - .string() - .notRequired() - .test( - 'metadata.uid', - 'The uid is not formatted according to schema', - value => value === undefined || value.length > 0, - ); - - const etagSchema = yup - .string() - .notRequired() - .test( - 'metadata.etag', - 'The etag value is not according to schema', - value => value === undefined || value.length > 0, - ); - - const generationSchema = yup - .number() - .notRequired() - .test( - 'metadata.generation', - 'The generation value is not according to schema', - value => value === undefined || value > 0, - ); - - const nameSchema = yup - .string() - .notRequired() - .test( - 'metadata.name', - 'The name is not formatted according to schema', - value => value === undefined || validators.isValidEntityName(value), - ); - - const namespaceSchema = yup - .string() - .notRequired() - .test( - 'metadata.namespace', - 'The namespace is malformed', - value => value === undefined || validators.isValidNamespace(value), - ); - - const labelsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.labels.keys', - message: 'Label keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidLabelKey) - ); - }, - }) - .test({ - name: 'metadata.labels.values', - message: 'Label values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidLabelValue) - ); - }, - }); - - const annotationsSchema = yup - .object>() - .notRequired() - .test({ - name: 'metadata.annotations.keys', - message: 'Annotation keys not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.keys(value).every(validators.isValidAnnotationKey) - ); - }, - }) - .test({ - name: 'metadata.annotations.values', - message: 'Annotation values not formatted according to schema', - test(value: object) { - return ( - value === undefined || - Object.values(value).every(validators.isValidAnnotationValue) - ); - }, - }); - - const metadataSchema = yup - .object({ - uid: uidSchema, - etag: etagSchema, - generation: generationSchema, - name: nameSchema, - namespace: namespaceSchema, - labels: labelsSchema, - annotations: annotationsSchema, - }) - .notRequired(); - - const specSchema = yup.object({}).notRequired(); - - this.schema = yup - .object({ - apiVersion: apiVersionSchema, - kind: kindSchema, - metadata: metadataSchema, - spec: specSchema, - }) - .noUnknown(); - } - - async parse(data: any): Promise { - let result: DescriptorEnvelope; - try { - result = await this.schema.validate(data, { strict: true }); - } catch (e) { - throw new Error(`Malformed envelope, ${e}`); - } - - // These are keys with specific semantic meaning in a document, that we do - // not want to appear in the root of the spec, or as labels or as - // annotations, because they will lead to confusion. - const reservedKeys = [ - 'apiVersion', - 'kind', - 'uid', - 'etag', - 'generation', - 'name', - 'namespace', - 'labels', - 'annotations', - 'spec', - ]; - for (const key of reservedKeys) { - if (result.spec?.hasOwnProperty(key)) { - throw new Error( - `The spec may not contain the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.labels?.hasOwnProperty(key)) { - throw new Error( - `A label may not have the key ${key}, because it has reserved meaning`, - ); - } - if (result.metadata?.annotations?.hasOwnProperty(key)) { - throw new Error( - `An annotation may not have the key ${key}, because it has reserved meaning`, - ); - } - } - - return result; - } -} diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index ca6f2dd2be..b6aceaecdd 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ -export * from './DescriptorParsers'; -export * from './LocationReaders'; -export * from './types'; +export * from './descriptor'; +export { IngestionModels } from './IngestionModels'; +export * from './source'; +export type { IngestionModel } from './types'; diff --git a/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts new file mode 100644 index 0000000000..a670f309ad --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileLocationReader } from './readers/FileLocationReader'; +import { GitHubLocationReader } from './readers/GitHubLocationReader'; +import { LocationReader } from './readers/types'; + +export class LocationReaders implements LocationReader { + private readonly readers: LocationReader[]; + + static defaultReaders(): LocationReader[] { + return [new FileLocationReader(), new GitHubLocationReader()]; + } + + constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) { + this.readers = readers; + } + + async tryRead(type: string, target: string): Promise { + for (const reader of this.readers) { + const result = await reader.tryRead(type, target); + if (result) { + return result; + } + } + throw new Error(`Could not read unknown location "${type}", "${target}"`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/index.ts b/plugins/catalog-backend/src/ingestion/source/index.ts new file mode 100644 index 0000000000..db7aa2f0bd --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { LocationReaders } from './LocationReaders'; +export { FileLocationReader } from './readers/FileLocationReader'; +export { GitHubLocationReader } from './readers/GitHubLocationReader'; +export type { LocationReader } from './readers/types'; diff --git a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts similarity index 63% rename from plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts rename to plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts index 9d2794ec4f..0c64aebf14 100644 --- a/plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts @@ -15,22 +15,21 @@ */ import fs from 'fs-extra'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; +import { LocationReader } from './types'; + +/** + * Reads a file from the local file system. + */ +export class FileLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'file') { + return undefined; + } -export class FileLocationSource implements LocationSource { - async read(target: string): Promise { - let rawYaml; try { - rawYaml = await fs.readFile(target, 'utf8'); + return await fs.readFile(target); } catch (e) { throw new Error(`Unable to read "${target}", ${e}`); } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } } } diff --git a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts similarity index 64% rename from plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts rename to plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts index 083c2f7e86..1f0f6ed539 100644 --- a/plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts @@ -16,43 +16,26 @@ jest.mock('node-fetch'); -import fs from 'fs-extra'; import fetch from 'node-fetch'; -import path from 'path'; -import { GitHubLocationSource } from '../GitHubLocationSource'; +import { GitHubLocationReader } from './GitHubLocationReader'; const { Response } = jest.requireActual('node-fetch'); -const FIXTURES_DIR = path.resolve( - __dirname, - '..', - '..', - '..', - '..', - 'fixtures', -); -const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => { - acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8'); - return acc; -}, {} as Record); - -describe('Unit: GitHubLocationSource', () => { +describe('Unit: GitHubLocationReader', () => { beforeEach(() => { jest.clearAllMocks(); }); it('fetches the file and parses it correctly', async () => { - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures['one_component.yaml'])), - ); - const reader = new GitHubLocationSource(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); - const result = await reader.read( + const reader = new GitHubLocationReader(); + const buffer = await reader.tryRead( + 'github', 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', ); - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); + expect(buffer?.toString('utf8')).toBe('hello'); }); it('changes the url to point to https://raw.githubusercontent.com', async () => { @@ -61,12 +44,12 @@ describe('Unit: GitHubLocationSource', () => { const folderPath = `master/plugins/catalog-backend/fixtures`; const componentFilename = `one_component.yaml`; const rawGitHubUrl = `https://raw.githubusercontent.com`; - const reader = new GitHubLocationSource(); - (fetch as any).mockReturnValueOnce( - Promise.resolve(new Response(fixtures[componentFilename])), - ); - await reader.read( + const reader = new GitHubLocationReader(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + await reader.tryRead( + 'github', `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, ); @@ -76,7 +59,7 @@ describe('Unit: GitHubLocationSource', () => { }); describe('rejects wrong urls', () => { - const reader = new GitHubLocationSource(); + const reader = new GitHubLocationReader(); it.each([ ['http://example.com/one_component.yaml'], @@ -87,7 +70,7 @@ describe('Unit: GitHubLocationSource', () => { ])( '%p', async (url: string) => - await expect(reader.read(url)).rejects.toThrow(/url/), + await expect(reader.tryRead('github', url)).rejects.toThrow(/url/), ); }); }); @@ -100,11 +83,10 @@ describe('Integration: GitHubLocationSource', () => { it('fetches the fixture from backstage repo', async () => { const PERMANENT_LINK = 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; - const reader = new GitHubLocationSource(); - const result = await reader.read(PERMANENT_LINK); + const reader = new GitHubLocationReader(); + const result = await reader.tryRead('github', PERMANENT_LINK); - expect(result[0].type).toBe('data'); - expect((result[0] as any).data.metadata.name).toBe('component3'); + expect(result?.toString('utf8')).toContain('component3'); }); }); diff --git a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts similarity index 70% rename from plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts rename to plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts index 69ba4911a6..bd330e28b4 100644 --- a/plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts @@ -16,17 +16,31 @@ import fetch from 'node-fetch'; import { URL } from 'url'; -import { LocationSource, ReaderOutput } from '../types'; -import { readDescriptorYaml } from './util'; +import { LocationReader } from './types'; -// Pointing to raw.githubusercontent.com for now -// to be changed in the future, after auth and tokens are done -export class GitHubLocationSource implements LocationSource { - async read(target: string): Promise { - let url: URL; +/** + * Reads a file whose target is a GitHub URL. + * + * Uses raw.githubusercontent.com for now, but this will probably change in the + * future when token auth is implemented. + */ +export class GitHubLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'github') { + return undefined; + } + const url = this.buildRawUrl(target); try { - url = new URL(target); + return await fetch(url.toString()).then(x => x.buffer()); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } + + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); const [ empty, @@ -51,23 +65,10 @@ export class GitHubLocationSource implements LocationSource { url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); url.hostname = 'raw.githubusercontent.com'; url.protocol = 'https'; + + return url; } catch (e) { throw new Error(`Incorrect url: ${target}, ${e}`); } - - let rawYaml; - try { - rawYaml = await fetch(url.toString()).then(x => { - return x.text(); - }); - } catch (e) { - throw new Error(`Unable to read "${target}", ${e}`); - } - - try { - return readDescriptorYaml(rawYaml); - } catch (e) { - throw new Error(`Malformed descriptor at "${target}", ${e}`); - } } } diff --git a/packages/core/src/icons/types.ts b/plugins/catalog-backend/src/ingestion/source/readers/types.ts similarity index 54% rename from packages/core/src/icons/types.ts rename to plugins/catalog-backend/src/ingestion/source/readers/types.ts index 599cb969df..37c7b46885 100644 --- a/packages/core/src/icons/types.ts +++ b/plugins/catalog-backend/src/ingestion/source/readers/types.ts @@ -14,8 +14,16 @@ * limitations under the License. */ -import { ComponentType } from 'react'; -import { SvgIconProps } from '@material-ui/core'; -export type IconComponent = ComponentType; -export type SystemIconKey = 'user' | 'group'; -export type SystemIcons = { [key in SystemIconKey]: IconComponent }; +export type LocationReader = { + /** + * Reads the contents of a single location. + * + * @param type The type of location to read + * @param target The location target (type-specific) + * @returns The target contents, as a raw Buffer, or undefined if this type + * was not meant to be consumed by this reader + * @throws An error if the type was meant for this reader, but could not be + * read + */ + tryRead(type: string, target: string): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/sources/util.ts b/plugins/catalog-backend/src/ingestion/sources/util.ts deleted file mode 100644 index cccbeb92b3..0000000000 --- a/plugins/catalog-backend/src/ingestion/sources/util.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import yaml from 'yaml'; -import { ReaderOutput } from '../types'; - -export function readDescriptorYaml(data: string): ReaderOutput[] { - let documents; - try { - documents = yaml.parseAllDocuments(data); - } catch (e) { - throw new Error(`Could not parse YAML data, ${e}`); - } - - const result: ReaderOutput[] = []; - - for (const document of documents) { - if (document.contents) { - if (document.errors?.length) { - result.push({ - type: 'error', - error: new Error(`Malformed YAML document, ${document.errors[0]}`), - }); - } else { - const json = document.toJSON(); - if (typeof json !== 'object' || Array.isArray(json)) { - result.push({ - type: 'error', - error: new Error(`Malformed descriptor, expected object at root`), - }); - } else { - result.push({ - type: 'data', - data: json, - }); - } - } - } - } - - return result; -} diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index e5083a65f1..8878c2af5b 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -14,172 +14,8 @@ * limitations under the License. */ -import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser'; +import { ReaderOutput } from './descriptor/parsers/types'; -export type ComponentDescriptor = ComponentDescriptorV1beta1; - -/** - * Metadata fields common to all versions/kinds of entity. - * - * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta - */ -export type EntityMeta = { - /** - * A globally unique ID for the entity. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, but the server is free to reject requests - * that do so in such a way that it breaks semantics. - */ - uid?: string; - - /** - * An opaque string that changes for each update operation to any part of - * the entity, including metadata. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. The field can (optionally) be specified when performing - * update or delete operations, and the server will then reject the - * operation if it does not match the current stored value. - */ - etag?: string; - - /** - * A positive nonzero number that indicates the current generation of data - * for this entity; the value is incremented each time the spec changes. - * - * This field can not be set by the user at creation time, and the server - * will reject an attempt to do so. The field will be populated in read - * operations. - */ - generation?: number; - - /** - * The name of the entity. - * - * Must be uniqe within the catalog at any given point in time, for any - * given namespace, for any given kind. - */ - name?: string; - - /** - * The namespace that the entity belongs to. - */ - namespace?: string; - - /** - * Key/value pairs of identifying information attached to the entity. - */ - labels?: Record; - - /** - * Key/value pairs of non-identifying auxiliary information attached to the - * entity. - */ - annotations?: Record; -}; - -/** - * The format envelope that's common to all versions/kinds. - * - * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ - */ -export type DescriptorEnvelope = { - /** - * The version of specification format for this particular entity that - * this is written against. - */ - apiVersion: string; - - /** - * The high level entity type being described. - */ - kind: string; - - /** - * Optional metadata related to the entity. - */ - metadata?: EntityMeta; - - /** - * The specification data describing the entity itself. - */ - spec?: object; -}; - -/** - * Parses and validates descriptors. - * - * The output must be validated and well formed. - */ -export type DescriptorParser = { - /** - * Parses and validates a single raw descriptor. - * - * @param descriptor A raw descriptor object - * @returns A structure describing the parsed and validated descriptor - * @throws An Error if the descriptor was malformed - */ - parse(descriptor: object): Promise; -}; - -/** - * Parses and validates a single envelope into its materialized kind. - * - * These parsers may assume that the envelope is already validated and well - * formed. - */ -export type KindParser = { - /** - * Try to parse an envelope into a materialized kind. - * - * @param envelope A valid descriptor envelope - * @returns A materialized type, or undefined if the given version/kind is - * not meant to be handled by this parser - * @throws An Error if the type was handled and found to not be properly - * formatted - */ - tryParse( - envelope: DescriptorEnvelope, - ): Promise; -}; - -export class ParserError extends Error { - constructor(message?: string, private _entityName?: string | undefined) { - super(message); - } - get entityName() { - return this._entityName; - } -} - -export type ReaderOutput = - | { type: 'error'; error: Error } - | { type: 'data'; data: object }; - -export type LocationReader = { - /** - * Reads the contents of a single location. - * - * @param type The type of location to read - * @param target The location target (type-specific) - * @returns The parsed contents, as an array of unverified descriptors or - * errors where the individual documents could not be parsed. - * @throws An error if the location as a whole could not be read - */ - read(type: string, target: string): Promise; -}; - -export type LocationSource = { - /** - * Reads the contents of a single location. - * - * @param target The location target to read - * @returns The parsed contents, as an array of unverified descriptors - * @throws An error if the location target could not be read - */ - read(target: string): Promise; +export type IngestionModel = { + readLocation(type: string, target: string): Promise; }; diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 8c29362015..f5675e6516 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -15,10 +15,10 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog'; -import { DescriptorEnvelope } from '../ingestion'; import { createRouter } from './router'; class MockEntitiesCatalog implements EntitiesCatalog { @@ -37,7 +37,9 @@ class MockLocationsCatalog implements LocationsCatalog { describe('createRouter', () => { describe('entities', () => { it('happy path: lists entities', async () => { - const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }]; + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; const catalog = new MockEntitiesCatalog(); catalog.entities.mockResolvedValueOnce(entities); @@ -76,7 +78,7 @@ describe('createRouter', () => { describe('entityByUid', () => { it('can fetch entity by uid', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { @@ -117,7 +119,7 @@ describe('createRouter', () => { describe('entityByName', () => { it('can fetch entity by name', async () => { - const entity: DescriptorEnvelope = { + const entity: Entity = { apiVersion: 'a', kind: 'b', metadata: { diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index c3318a7377..0678cd3aef 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -22,7 +22,7 @@ import { addLocationSchema, EntitiesCatalog, EntityFilters, - LocationsCatalog, + LocationsCatalog } from '../catalog'; import { validateRequestBody } from './util'; diff --git a/plugins/catalog-backend/src/setupTests.ts b/plugins/catalog-backend/src/setupTests.ts index 3fa7cb04b4..a3b2b7e123 100644 --- a/plugins/catalog-backend/src/setupTests.ts +++ b/plugins/catalog-backend/src/setupTests.ts @@ -15,3 +15,4 @@ */ require('jest-fetch-mock').enableMocks(); +export {}; diff --git a/plugins/circleci/src/api/index.ts b/plugins/circleci/src/api/index.ts index 26b0213e3a..64a33e4c13 100644 --- a/plugins/circleci/src/api/index.ts +++ b/plugins/circleci/src/api/index.ts @@ -28,7 +28,8 @@ import { } from 'circleci-api'; import { createApiRef } from '@backstage/core'; -export { BuildWithSteps, BuildStepAction, BuildSummary, GitType }; +export { GitType }; +export type { BuildWithSteps, BuildStepAction, BuildSummary }; export const circleCIApiRef = createApiRef({ id: 'plugin.circleci.service', diff --git a/plugins/identity-backend/src/index.test.ts b/plugins/identity-backend/src/index.test.ts index b3e2f19771..4fca4ca746 100644 --- a/plugins/identity-backend/src/index.test.ts +++ b/plugins/identity-backend/src/index.test.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import '@backstage/backend-common'; + describe('test', () => { it('unbreaks the test runner', () => { expect(true).toBeTruthy(); diff --git a/plugins/identity-backend/src/setupTests.ts b/plugins/identity-backend/src/setupTests.ts index 3fa7cb04b4..a3b2b7e123 100644 --- a/plugins/identity-backend/src/setupTests.ts +++ b/plugins/identity-backend/src/setupTests.ts @@ -15,3 +15,4 @@ */ require('jest-fetch-mock').enableMocks(); +export {}; diff --git a/yarn.lock b/yarn.lock index 74b9e451b0..8439f5b0b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3071,7 +3071,7 @@ regenerator-runtime "^0.13.3" util-deprecate "^1.0.2" -"@storybook/addons@5.3.18", "@storybook/addons@^5.3.17": +"@storybook/addons@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/addons/-/addons-5.3.18.tgz#5cbba6407ef7a802041c5ee831473bc3bed61f64" integrity sha512-ZQjDgTUDFRLvAiBg2d8FgPgghfQ+9uFyXQbtiGlTBLinrPCeQd7J86qiUES0fcGoohCCw0wWKtvB0WF2z1XNDg== @@ -3084,6 +3084,19 @@ global "^4.3.2" util-deprecate "^1.0.2" +"@storybook/addons@^5.3.17": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/addons/-/addons-5.3.19.tgz#3a7010697afd6df9a41b8c8a7351d9a06ff490a4" + integrity sha512-Ky/k22p6i6FVNvs1VhuFyGvYJdcp+FgXqFgnPyY/OXJW/vPDapdElpTpHJZLFI9I2FQBDcygBPU5RXkumQ+KUQ== + dependencies: + "@storybook/api" "5.3.19" + "@storybook/channels" "5.3.19" + "@storybook/client-logger" "5.3.19" + "@storybook/core-events" "5.3.19" + core-js "^3.0.1" + global "^4.3.2" + util-deprecate "^1.0.2" + "@storybook/api@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/api/-/api-5.3.18.tgz#95582ab90d947065e0e34ed603650a3630dcbd16" @@ -3110,6 +3123,32 @@ telejson "^3.2.0" util-deprecate "^1.0.2" +"@storybook/api@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/api/-/api-5.3.19.tgz#77f15e9e2eee59fe1ddeaba1ef39bc34713a6297" + integrity sha512-U/VzDvhNCPmw2igvJYNNM+uwJCL+3teiL6JmuoL4/cmcqhI6IqqG9dZmMP1egoCd19wXEP7rnAfB/VcYVg41dQ== + dependencies: + "@reach/router" "^1.2.1" + "@storybook/channels" "5.3.19" + "@storybook/client-logger" "5.3.19" + "@storybook/core-events" "5.3.19" + "@storybook/csf" "0.0.1" + "@storybook/router" "5.3.19" + "@storybook/theming" "5.3.19" + "@types/reach__router" "^1.2.3" + core-js "^3.0.1" + fast-deep-equal "^2.0.1" + global "^4.3.2" + lodash "^4.17.15" + memoizerific "^1.11.3" + prop-types "^15.6.2" + react "^16.8.3" + semver "^6.0.0" + shallow-equal "^1.1.0" + store2 "^2.7.1" + telejson "^3.2.0" + util-deprecate "^1.0.2" + "@storybook/channel-postmessage@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/channel-postmessage/-/channel-postmessage-5.3.18.tgz#93d46740b5cc9b36ddd073f0715b54c4959953bf" @@ -3128,6 +3167,13 @@ dependencies: core-js "^3.0.1" +"@storybook/channels@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/channels/-/channels-5.3.19.tgz#65ad7cd19d70aa5eabbb2e5e39ceef5e510bcb7f" + integrity sha512-38seaeyshRGotTEZJppyYMg/Vx2zRKgFv1L6uGqkJT0LYoNSYtJhsiNFCJ2/KUJu2chAJ/j8h80bpVBVLQ/+WA== + dependencies: + core-js "^3.0.1" + "@storybook/client-api@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/client-api/-/client-api-5.3.18.tgz#e71041796f95888de0e4524734418e6b120b060a" @@ -3158,6 +3204,13 @@ dependencies: core-js "^3.0.1" +"@storybook/client-logger@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-5.3.19.tgz#fbbd186e82102eaca1d6a5cca640271cae862921" + integrity sha512-nHftT9Ow71YgAd2/tsu79kwKk30mPuE0sGRRUHZVyCRciGFQweKNOS/6xi2Aq+WwBNNjPKNlbgxwRt1yKe1Vkg== + dependencies: + core-js "^3.0.1" + "@storybook/components@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/components/-/components-5.3.18.tgz#528f6ab1660981e948993a04b407a6fad7751589" @@ -3192,6 +3245,13 @@ dependencies: core-js "^3.0.1" +"@storybook/core-events@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/core-events/-/core-events-5.3.19.tgz#18020cd52e0d8ef0973a8e9622a10d5f99796f79" + integrity sha512-lh78ySqMS7pDdMJAQAe35d1I/I4yPTqp09Cq0YIYOxx9BQZhah4DZTV1QIZt22H5p2lPb5MWLkWSxBaexZnz8A== + dependencies: + core-js "^3.0.1" + "@storybook/core@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/core/-/core-5.3.18.tgz#3f3c0498275826c1cc4368aba203ac17a6ae5c9c" @@ -3332,6 +3392,21 @@ qs "^6.6.0" util-deprecate "^1.0.2" +"@storybook/router@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/router/-/router-5.3.19.tgz#0f783b85658f99e4007f74347ad7ef17dbf7fc3a" + integrity sha512-yNClpuP7BXQlBTRf6Ggle3/R349/k6kvI5Aim4jf6X/2cFVg2pzBXDAF41imNm9PcvdxwabQLm6I48p7OvKr/w== + dependencies: + "@reach/router" "^1.2.1" + "@storybook/csf" "0.0.1" + "@types/reach__router" "^1.2.3" + core-js "^3.0.1" + global "^4.3.2" + lodash "^4.17.15" + memoizerific "^1.11.3" + qs "^6.6.0" + util-deprecate "^1.0.2" + "@storybook/source-loader@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/source-loader/-/source-loader-5.3.18.tgz#39ba28d9664ab8204d6b04ee757772369931e7e5" @@ -3366,6 +3441,24 @@ resolve-from "^5.0.0" ts-dedent "^1.1.0" +"@storybook/theming@5.3.19": + version "5.3.19" + resolved "https://registry.npmjs.org/@storybook/theming/-/theming-5.3.19.tgz#177d9819bd64f7a1a6ea2f1920ffa5baf9a5f467" + integrity sha512-ecG+Rq3hc1GOzKHamYnD4wZ0PEP9nNg0mXbC3RhbxfHj+pMMCWWmx9B2Uu75SL1PTT8WcfkFO0hU/0IO84Pzlg== + dependencies: + "@emotion/core" "^10.0.20" + "@emotion/styled" "^10.0.17" + "@storybook/client-logger" "5.3.19" + core-js "^3.0.1" + deep-object-diff "^1.1.0" + emotion-theming "^10.0.19" + global "^4.3.2" + memoizerific "^1.11.3" + polished "^3.3.1" + prop-types "^15.7.2" + resolve-from "^5.0.0" + ts-dedent "^1.1.0" + "@storybook/ui@5.3.18": version "5.3.18" resolved "https://registry.npmjs.org/@storybook/ui/-/ui-5.3.18.tgz#c66f6d94a3c50bb706f4d5b1d5592439110f16f0" @@ -14189,9 +14282,9 @@ marked@^0.8.0: integrity sha512-tZfJS8uE0zpo7xpTffwFwYRfW9AzNcdo04Qcjs+C9+oCy8MSRD2reD5iDVtYx8mtLaqsGughw/YLlcwNxAHA1g== material-table@^1.58.0: - version "1.58.0" - resolved "https://registry.npmjs.org/material-table/-/material-table-1.58.0.tgz#1902f88b74436ce880b234c7c713f0e34a7e2f9d" - integrity sha512-5xiiKERNZv9Zai2TMT5d9LRLrKlofAFe2YZZzxP7cJ9DIZC8HcaflunyIVHYTzK1yFS51TqjDvHIqxYLIqdahQ== + version "1.58.2" + resolved "https://registry.npmjs.org/material-table/-/material-table-1.58.2.tgz#dc0d19652848e6bb92f747d122bd7d4681cca6dc" + integrity sha512-s/m6ebyXFXmg07zxv1Fl6qPySKaiQhASXaOB3ubRKUFA1DkryUy3PGSEVWTjUYnRyi63kGq+N6b5nsokLR6m5A== dependencies: "@date-io/date-fns" "^1.1.0" "@material-ui/pickers" "^3.2.2" @@ -18995,10 +19088,10 @@ source-map-resolve@^0.5.0, source-map-resolve@^0.5.2: source-map-url "^0.4.0" urix "^0.1.0" -source-map-support@^0.5.6, source-map-support@~0.5.12: - version "0.5.16" - resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz#0ae069e7fe3ba7538c64c98515e35339eac5a042" - integrity sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ== +source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.12: + version "0.5.19" + resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" + integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -20323,14 +20416,14 @@ ts-loader@^7.0.4: semver "^6.0.0" ts-node@^8.6.2: - version "8.8.1" - resolved "https://registry.npmjs.org/ts-node/-/ts-node-8.8.1.tgz#7c4d3e9ed33aa703b64b28d7f9d194768be5064d" - integrity sha512-10DE9ONho06QORKAaCBpPiFCdW+tZJuY/84tyypGtl6r+/C7Asq0dhqbRZURuUlLQtZxxDvT8eoj8cGW0ha6Bg== + version "8.10.2" + resolved "https://registry.npmjs.org/ts-node/-/ts-node-8.10.2.tgz#eee03764633b1234ddd37f8db9ec10b75ec7fb8d" + integrity sha512-ISJJGgkIpDdBhWVu3jufsWpK3Rzo7bdiIXJjQc0ynKxVOVcg2oIrf2H2cejminGrptVc6q6/uynAHNCuWGbpVA== dependencies: arg "^4.1.0" diff "^4.0.1" make-error "^1.1.1" - source-map-support "^0.5.6" + source-map-support "^0.5.17" yn "3.1.1" ts-pnp@^1.1.2: