diff --git a/packages/catalog-model/.eslintrc.js b/packages/catalog-model/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/packages/catalog-model/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/packages/catalog-model/README.md b/packages/catalog-model/README.md new file mode 100644 index 0000000000..755b9ee63c --- /dev/null +++ b/packages/catalog-model/README.md @@ -0,0 +1,12 @@ +# Catalog Model + +Contains the core model types and validators/policies used by the Backstage catalog functionality. + +This package will be imported both by the frontend and backend parts of the catalog, +as well as by others that want to consume catalog data. + +## Links + +- (Default frontend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog] +- (Default backend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog-backend] +- (The Backstage homepage)[https://backstage.io] diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json new file mode 100644 index 0000000000..ffd8fdba22 --- /dev/null +++ b/packages/catalog-model/package.json @@ -0,0 +1,34 @@ +{ + "name": "@backstage/catalog-model", + "version": "0.1.1-alpha.6", + "main": "dist/index.esm.js", + "main:src": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "lodash": "^4.17.15", + "yup": "^0.28.5" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.6", + "@types/jest": "^25.2.2", + "@types/lodash": "^4.14.151", + "@types/yup": "^0.28.2", + "yaml": "^1.9.2" + }, + "files": [ + "dist/**/*.{js,d.ts}" + ] +} diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts new file mode 100644 index 0000000000..cedf0df13d --- /dev/null +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + FieldFormatEntityPolicy, + ForeignRootFieldsEntityPolicy, + ReservedFieldsEntityPolicy, + SchemaValidEntityPolicy, +} from './entity'; +import { ComponentV1beta1Policy } from './kinds'; +import { EntityPolicy } from './types'; + +// Helper that requires that all of a set of policies can be successfully +// applied +class AllEntityPolicies implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + let result = entity; + for (const policy of this.policies) { + result = await policy.apply(entity); + } + return result; + } +} + +// Helper that requires that at least one of a set of policies can be +// successfully applied +class AnyEntityPolicy implements EntityPolicy { + constructor(private readonly policies: EntityPolicy[]) {} + + async apply(entity: Entity): Promise { + for (const policy of this.policies) { + try { + return await policy.apply(entity); + } catch { + continue; + } + } + throw new Error(`The entity did not match any known policy`); + } +} + +export class EntityPolicies implements EntityPolicy { + private readonly policy: EntityPolicy; + + static defaultPolicies(): EntityPolicy { + return EntityPolicies.allOf([ + EntityPolicies.allOf([ + new SchemaValidEntityPolicy(), + new ForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy(), + new ReservedFieldsEntityPolicy(), + ]), + EntityPolicies.anyOf([new ComponentV1beta1Policy()]), + ]); + } + + static allOf(policies: EntityPolicy[]): EntityPolicy { + return new AllEntityPolicies(policies); + } + + static anyOf(policies: EntityPolicy[]): EntityPolicy { + return new AnyEntityPolicy(policies); + } + + constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) { + this.policy = policy; + } + + apply(entity: Entity): Promise { + return this.policy.apply(entity); + } +} diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts new file mode 100644 index 0000000000..6f57626749 --- /dev/null +++ b/packages/catalog-model/src/entity/Entity.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The format envelope that's common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type Entity = { + /** + * The version of specification format for this particular entity that + * this is written against. + */ + apiVersion: string; + + /** + * The high level entity type being described. + */ + kind: string; + + /** + * Optional metadata related to the entity. + */ + metadata?: EntityMeta; + + /** + * The specification data describing the entity itself. + */ + spec?: object; +}; + +/** + * Metadata fields common to all versions/kinds of entity. + * + * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/ + */ +export type EntityMeta = { + /** + * A globally unique ID for the entity. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, but the server is free to reject requests + * that do so in such a way that it breaks semantics. + */ + uid?: string; + + /** + * An opaque string that changes for each update operation to any part of + * the entity, including metadata. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. The field can (optionally) be specified when performing + * update or delete operations, and the server will then reject the + * operation if it does not match the current stored value. + */ + etag?: string; + + /** + * A positive nonzero number that indicates the current generation of data + * for this entity; the value is incremented each time the spec changes. + * + * This field can not be set by the user at creation time, and the server + * will reject an attempt to do so. The field will be populated in read + * operations. + */ + generation?: number; + + /** + * The name of the entity. + * + * Must be uniqe within the catalog at any given point in time, for any + * given namespace, for any given kind. + */ + name?: string; + + /** + * The namespace that the entity belongs to. + */ + namespace?: string; + + /** + * Key/value pairs of identifying information attached to the entity. + */ + labels?: Record; + + /** + * Key/value pairs of non-identifying auxiliary information attached to the + * entity. + */ + annotations?: Record; +}; diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts new file mode 100644 index 0000000000..9e96021336 --- /dev/null +++ b/packages/catalog-model/src/entity/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { Entity, EntityMeta } from './Entity'; +export * from './policies'; diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts new file mode 100644 index 0000000000..d81b1155be --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; + +describe('FieldFormatEntityPolicy', () => { + let data: any; + let policy: FieldFormatEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new FieldFormatEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad apiVersion', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + data.apiVersion = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad kind', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + data.kind = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('handles missing metadata gracefully', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('handles missing spec gracefully', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name.*7/); + data.metadata.name = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/name.*aaaa/); + }); + + it('rejects bad namespace', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace.*7/); + data.metadata.namespace = 'a'.repeat(1000); + await expect(policy.apply(data)).rejects.toThrow(/namespace.*aaaa/); + }); + + it('rejects bad label key', async () => { + data.metadata.labels['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad label value', async () => { + data.metadata.labels.a = 'a#b'; + await expect(policy.apply(data)).rejects.toThrow(/label.*a#b/i); + }); + + it('rejects bad annotation key', async () => { + data.metadata.annotations['a#b'] = 'value'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*a#b/i); + }); + + it('rejects bad annotation value', async () => { + data.metadata.annotations.a = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*7/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts new file mode 100644 index 0000000000..1f94354f3f --- /dev/null +++ b/packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { makeValidator, Validators } from '../../validation'; +import { Entity } from '../Entity'; + +/** + * Ensures that the format of individual fields of the entity envelope + * is valid. + * + * This does not take into account machine generated fields such as uid, etag + * and generation. + */ +export class FieldFormatEntityPolicy implements EntityPolicy { + private readonly validators: Validators; + + constructor(validators: Validators = makeValidator()) { + this.validators = validators; + } + + async apply(entity: Entity): Promise { + function require( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + if (value === undefined || value === null) { + throw new Error(`${field} must have a value`); + } + + let isValid: boolean; + try { + isValid = validator(value); + } catch (e) { + throw new Error(`${field} could not be validated, ${e}`); + } + + if (!isValid) { + throw new Error(`${field} "${value}" is not valid`); + } + } + + function optional( + field: string, + value: any, + validator: (value: any) => boolean, + ) { + return value === undefined || require(field, value, validator); + } + + require('apiVersion', entity.apiVersion, this.validators.isValidApiVersion); + require('kind', entity.kind, this.validators.isValidKind); + + optional( + 'metadata.name', + entity.metadata?.name, + this.validators.isValidEntityName, + ); + optional( + 'metadata.namespace', + entity.metadata?.namespace, + this.validators.isValidNamespace, + ); + + for (const [k, v] of Object.entries(entity.metadata?.labels ?? [])) { + require(`labels.${k}`, k, this.validators.isValidLabelKey); + require(`labels.${k}`, v, this.validators.isValidLabelValue); + } + + for (const [k, v] of Object.entries(entity.metadata?.annotations ?? [])) { + require(`annotations.${k}`, k, this.validators.isValidAnnotationKey); + require(`annotations.${k}`, v, this.validators.isValidAnnotationValue); + } + + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..98299259f8 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; + +describe('ForeignRootFieldsEntityPolicy', () => { + let data: any; + let policy: ForeignRootFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ForeignRootFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects unknown root fields', async () => { + data.spec2 = {}; + await expect(policy.apply(data)).rejects.toThrow(/spec2/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts new file mode 100644 index 0000000000..a4733e9a42 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ForeignRootFieldsEntityPolicy.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec']; + +/** + * Ensures that there are no foreign root fields in the entity. + */ +export class ForeignRootFieldsEntityPolicy implements EntityPolicy { + private readonly knownFields: string[]; + + constructor(knownFields: string[] = defaultKnownFields) { + this.knownFields = knownFields; + } + + async apply(entity: Entity): Promise { + for (const field of Object.keys(entity)) { + if (!this.knownFields.includes(field)) { + throw new Error(`Unknown field ${field}`); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts new file mode 100644 index 0000000000..348eabbdac --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; + +describe('ReservedFieldsEntityPolicy', () => { + let data: any; + let policy: ReservedFieldsEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new ReservedFieldsEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects reserved keys in the spec root', async () => { + data.spec.apiVersion = 'a/b'; + await expect(policy.apply(data)).rejects.toThrow(/spec.*apiVersion/i); + }); + + it('rejects reserved keys in labels', async () => { + data.metadata.labels.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/label.*apiVersion/i); + }); + + it('rejects reserved keys in annotations', async () => { + data.metadata.annotations.apiVersion = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/annotation.*apiVersion/i); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts new file mode 100644 index 0000000000..be2f732ca4 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_RESERVED_ENTITY_FIELDS = [ + 'apiVersion', + 'kind', + 'uid', + 'etag', + 'generation', + 'name', + 'namespace', + 'labels', + 'annotations', + 'spec', +]; + +/** + * Ensures that fields are not given certain reserved names. + */ +export class ReservedFieldsEntityPolicy implements EntityPolicy { + private readonly reservedFields: string[]; + + constructor(fields?: string[]) { + this.reservedFields = [ + ...(fields ?? []), + ...DEFAULT_RESERVED_ENTITY_FIELDS, + ]; + } + + async apply(entity: Entity): Promise { + for (const field of this.reservedFields) { + if (entity.spec?.hasOwnProperty(field)) { + throw new Error( + `The spec may not contain the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.labels?.hasOwnProperty(field)) { + throw new Error( + `A label may not have the field ${field}, because it has reserved meaning`, + ); + } + if (entity.metadata?.annotations?.hasOwnProperty(field)) { + throw new Error( + `An annotation may not have the field ${field}, because it has reserved meaning`, + ); + } + } + return entity; + } +} diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts new file mode 100644 index 0000000000..b9d1165a60 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { Entity } from '../Entity'; +import { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; + +describe('SchemaValidEntityPolicy', () => { + let data: any; + let policy: SchemaValidEntityPolicy; + + beforeEach(() => { + data = yaml.parse(` + apiVersion: backstage.io/v1beta1 + kind: Component + metadata: + uid: e01199ab-08cc-44c2-8e19-5c29ded82521 + etag: lsndfkjsndfkjnsdfkjnsd== + generation: 13 + name: my-component-yay + namespace: the-namespace + labels: + backstage.io/custom: ValueStuff + annotations: + example.com/bindings: are-secret + spec: + custom: stuff + `); + policy = new SchemaValidEntityPolicy(); + }); + + it('works for the happy path', async () => { + await expect(policy.apply(data)).resolves.toBe(data); + }); + + // + // apiVersion and kind + // + + it('rejects wrong root type', async () => { + await expect(policy.apply((7 as unknown) as Entity)).rejects.toThrow( + /object/, + ); + }); + + it('rejects missing apiVersion', async () => { + delete data.apiVersion; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects bad apiVersion type', async () => { + data.apiVersion = 7; + await expect(policy.apply(data)).rejects.toThrow(/apiVersion/); + }); + + it('rejects missing kind', async () => { + delete data.kind; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + it('rejects bad kind type', async () => { + data.kind = 7; + await expect(policy.apply(data)).rejects.toThrow(/kind/); + }); + + // + // metadata + // + + it('accepts missing metadata', async () => { + delete data.medatata; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad metadata type', async () => { + data.metadata = 7; + await expect(policy.apply(data)).rejects.toThrow(/metadata/); + }); + + it('accepts missing uid', async () => { + delete data.metadata.uid; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad uid type', async () => { + data.metadata.uid = 7; + await expect(policy.apply(data)).rejects.toThrow(/uid/); + }); + + it('accepts missing etag', async () => { + delete data.metadata.etag; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad etag type', async () => { + data.metadata.etag = 7; + await expect(policy.apply(data)).rejects.toThrow(/etag/); + }); + + it('accepts missing generation', async () => { + delete data.metadata.generation; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad generation type', async () => { + data.metadata.generation = 'a'; + await expect(policy.apply(data)).rejects.toThrow(/generation/); + }); + + it('accepts missing name', async () => { + delete data.metadata.name; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad name type', async () => { + data.metadata.name = 7; + await expect(policy.apply(data)).rejects.toThrow(/name/); + }); + + it('accepts missing namespace', async () => { + delete data.metadata.namespace; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad namespace type', async () => { + data.metadata.namespace = 7; + await expect(policy.apply(data)).rejects.toThrow(/namespace/); + }); + + it('accepts missing labels', async () => { + delete data.metadata.labels; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad labels type', async () => { + data.metadata.labels = 7; + await expect(policy.apply(data)).rejects.toThrow(/labels/); + }); + + it('accepts missing annotations', async () => { + delete data.metadata.annotations; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects bad annotations type', async () => { + data.metadata.annotations = 7; + await expect(policy.apply(data)).rejects.toThrow(/annotations/); + }); + + // + // spec + // + + it('accepts missing spec', async () => { + delete data.spec; + await expect(policy.apply(data)).resolves.toBe(data); + }); + + it('rejects non-object spec', async () => { + data.spec = 7; + await expect(policy.apply(data)).rejects.toThrow(/spec/); + }); +}); diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts new file mode 100644 index 0000000000..7c0f5c20b6 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { EntityPolicy } from '../../types'; +import { Entity } from '../Entity'; + +const DEFAULT_ENTITY_SCHEMA = yup.object({ + apiVersion: yup.string().required(), + kind: yup.string().required(), + metadata: yup + .object({ + uid: yup + .string() + .notRequired() + .test( + 'metadata.uid', + 'The uid must not be empty', + value => value === undefined || value.length > 0, + ), + etag: yup + .string() + .notRequired() + .test( + 'metadata.etag', + 'The etag must not be empty', + value => value === undefined || value.length > 0, + ), + generation: yup + .number() + .notRequired() + .test( + 'metadata.generation', + 'The generation must be an integer greater than zero', + value => value === undefined || (value === (value | 0) && value > 0), + ), + name: yup.string().notRequired(), + namespace: yup.string().notRequired(), + labels: yup.object>().notRequired(), + annotations: yup.object>().notRequired(), + }) + .notRequired(), + spec: yup.object({}).notRequired(), +}); + +/** + * Ensures that the entity spec is valid according to a schema. + * + * This should be the first policy in the list, to ensure that other downstream + * policies can work with a structure that is at least valid in therms of the + * typescript type. + */ +export class SchemaValidEntityPolicy implements EntityPolicy { + private readonly schema: yup.Schema; + + constructor(schema: yup.Schema = DEFAULT_ENTITY_SCHEMA) { + this.schema = schema; + } + + async apply(entity: Entity): Promise { + try { + return await this.schema.validate(entity, { strict: true }); + } catch (e) { + throw new Error(`Malformed envelope, ${e}`); + } + } +} diff --git a/packages/catalog-model/src/entity/policies/index.ts b/packages/catalog-model/src/entity/policies/index.ts new file mode 100644 index 0000000000..f43aa68049 --- /dev/null +++ b/packages/catalog-model/src/entity/policies/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy'; +export { ForeignRootFieldsEntityPolicy } from './ForeignRootFieldsEntityPolicy'; +export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy'; +export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy'; diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts new file mode 100644 index 0000000000..fb51461053 --- /dev/null +++ b/packages/catalog-model/src/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './entity'; +export { EntityPolicies } from './EntityPolicies'; +export * from './kinds'; +export type { EntityPolicy } from './types'; +export * from './validation'; diff --git a/packages/catalog-model/src/kinds/ComponentV1beta1.ts b/packages/catalog-model/src/kinds/ComponentV1beta1.ts new file mode 100644 index 0000000000..b0a3f627a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/ComponentV1beta1.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import type { Entity, EntityMeta } from '../entity/Entity'; +import type { EntityPolicy } from '../types'; + +const API_VERSION = 'backstage.io/v1beta1'; +const KIND = 'Component'; + +export interface ComponentV1beta1 extends Entity { + apiVersion: typeof API_VERSION; + kind: typeof KIND; + metadata: EntityMeta & { + name: string; + }; + spec: { + type: string; + }; +} + +export class ComponentV1beta1Policy implements EntityPolicy { + private schema: yup.Schema; + + constructor() { + this.schema = yup.object>({ + metadata: yup + .object({ + name: yup.string().required(), + }) + .required(), + spec: yup + .object({ + type: yup.string().required(), + }) + .required(), + }); + } + + async apply(envelope: Entity): Promise { + if ( + envelope.apiVersion !== 'backstage.io/v1beta1' || + envelope.kind !== 'Component' + ) { + throw new Error('Unsupported apiVersion / kind'); + } + + return await this.schema.validate(envelope, { strict: true }); + } +} diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts new file mode 100644 index 0000000000..97d22c14a5 --- /dev/null +++ b/packages/catalog-model/src/kinds/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ComponentV1beta1 } from './ComponentV1beta1'; +export { ComponentV1beta1Policy } from './ComponentV1beta1'; +export { ComponentV1beta1 as Component }; +export { ComponentV1beta1 }; diff --git a/packages/catalog-model/src/setupTests.ts b/packages/catalog-model/src/setupTests.ts new file mode 100644 index 0000000000..f3b69cc361 --- /dev/null +++ b/packages/catalog-model/src/setupTests.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts new file mode 100644 index 0000000000..1d581cf23f --- /dev/null +++ b/packages/catalog-model/src/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from './entity/Entity'; + +/** + * A policy for validation or mutation to be applied to entities as they are + * entering the system. + */ +export type EntityPolicy = { + /** + * Applies validation or mutation on an entity. + * + * @param entity The entity, as validated/mutated so far in the policy tree + * @returns The incoming entity, or a mutated version of the same + * @throws An error if the entity should be rejected + */ + apply(entity: Entity): Promise; +}; diff --git a/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts new file mode 100644 index 0000000000..200e90b406 --- /dev/null +++ b/packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; + +describe('CommonValidatorFunctions', () => { + describe('isValidPrefixAndOrSuffix', () => { + it('only accepts strings', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + null, + '/', + () => true, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 7, + '/', + () => true, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + () => 'hello', + '/', + () => true, + () => true, + ), + ).toBe(false); + }); + + it('only accepts one or two parts', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b/c', + '/', + () => true, + () => true, + ), + ).toBe(false); + }); + + it('checks the prefix and suffix', () => { + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => true, + ), + ).toBe(true); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => false, + () => true, + ), + ).toBe(false); + expect( + CommonValidatorFunctions.isValidPrefixAndOrSuffix( + 'a/b', + '/', + () => true, + () => false, + ), + ).toBe(false); + }); + }); + + it.each([ + [null, true], + [undefined, false], + [1, true], + ['a', true], + [() => 'a', false], + [Symbol('a'), false], + [[], true], + [[1], true], + [[undefined], false], + [{}, true], + [{ a: 1 }, true], + [{ a: undefined }, false], + ] as [any, boolean][])(`isJsonSafe %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isJsonSafe(value)).toBe(result); + }); + + it.each([ + [null, false], + [7, false], + ['', false], + ['a', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + ['adam.bertil.caesar', true], + ['adam.ber-til.caesar', true], + ['adam.-bertil.caesar', false], + ['adam.bertil-.caesar', false], + ['adam/bertil.caesar', false], + [`a.${'b'.repeat(63)}.c`, true], + [`a.${'b'.repeat(64)}.c`, false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(61)}`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(62)}`, + false, + ], + ])(`isValidDnsSubdomain %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isValidDnsSubdomain(value)).toBe(result); + }); + + it.each([ + [null, false], + [7, false], + ['', false], + ['a', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + [`${'a'.repeat(63)}`, true], + [`${'a'.repeat(64)}`, false], + ])(`isValidDnsLabel %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.isValidDnsLabel(value)).toBe(result); + }); + + it.each([ + ['', ''], + ['a', 'a'], + ['a-b', 'ab'], + ['-a-b', 'ab'], + ['a_b', 'ab'], + [`${'a'.repeat(6000)}`, `${'a'.repeat(6000)}`], + ['_:;>!"#€', ''], + ])(`normalizeToLowercaseAlphanum %p ? %p`, (value, result) => { + expect(CommonValidatorFunctions.normalizeToLowercaseAlphanum(value)).toBe( + result, + ); + }); +}); diff --git a/packages/catalog-model/src/validation/CommonValidatorFunctions.ts b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts new file mode 100644 index 0000000000..96a91aca06 --- /dev/null +++ b/packages/catalog-model/src/validation/CommonValidatorFunctions.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import lodash from 'lodash'; + +/** + * Contains various helper validation and normalization functions that can be + * composed to form a Validator. + */ +export class CommonValidatorFunctions { + /** + * Checks that the value is on the form or , and validates + * those parts separately. + * + * @param value The value to check + * @param separator The separator between parts + * @param isValidPrefix Checks that the part before the separator is valid, if present + * @param isValidSuffix Checks that the part after the separator (or the entire value if there is no separator) is valid + */ + static isValidPrefixAndOrSuffix( + value: any, + separator: string, + isValidPrefix: (value: string) => boolean, + isValidSuffix: (value: string) => boolean, + ): boolean { + if (typeof value !== 'string') { + return false; + } + + const parts = value.split(separator); + if (parts.length === 1) { + return isValidSuffix(parts[0]); + } else if (parts.length === 2) { + return isValidPrefix(parts[0]) && isValidSuffix(parts[1]); + } + + return false; + } + + /** + * Checks that the value can be safely transferred as JSON. + * + * @param value The value to check + */ + static isJsonSafe(value: any): boolean { + try { + return lodash.isEqual(value, JSON.parse(JSON.stringify(value))); + } catch { + return false; + } + } + + /** + * Checks that the value is a valid DNS subdomain name. + * + * @param value The value to check + * @see https://tools.ietf.org/html/rfc1123 + */ + static isValidDnsSubdomain(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 253 && + value.split('.').every(CommonValidatorFunctions.isValidDnsLabel) + ); + } + + /** + * Checks that the value is a valid DNS label. + * + * @param value The value to check + * @see https://tools.ietf.org/html/rfc1123 + */ + static isValidDnsLabel(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-z0-9]+(\-[a-z0-9]+)*$/.test(value) + ); + } + + /** + * Normalizes by keeping only a-z, A-Z, and 0-9; and converts to lowercase. + * + * @param value The value to normalize + */ + static normalizeToLowercaseAlphanum(value: string): string { + return value + .split('') + .filter(x => /[a-zA-Z0-9]/.test(x)) + .join('') + .toLowerCase(); + } +} diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts new file mode 100644 index 0000000000..d0673085b4 --- /dev/null +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts @@ -0,0 +1,209 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; + +describe('KubernetesValidatorFunctions', () => { + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a-b', false], + ['a_b', false], + ['a.b', false], + ['a/a', true], + ['a/aAb5C', true], + ['a-b.c/v1', true], + ['a--b.c/v1', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/v1`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/v1`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidApiVersion %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidApiVersion(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['9AZ', false], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a-b', false], + ])(`isValidKind %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidKind(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ])(`isValidObjectName %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', false], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', false], + ['a.b', false], + ])(`isValidNamespace %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidNamespace(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ['a/a', true], + ['a-b.c/a', true], + ['a--b.c/a', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/a`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/a`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidLabelKey %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidLabelKey(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', true], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', false], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ])(`isValidLabelValue %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches); + }); + + it.each([ + [7, false], + [null, false], + ['', false], + ['a', true], + ['AZ09', true], + ['a'.repeat(63), true], + ['a'.repeat(64), false], + ['a/b', true], + ['a-b', true], + ['-a-b', false], + ['a-b-', false], + ['a--b', false], + ['a_b', true], + ['a.b', true], + ['a/a', true], + ['a-b.c/a', true], + ['a--b.c/a', false], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 61, + )}/a`, + true, + ], + [ + `${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat( + 62, + )}/a`, + false, + ], + [`a/${'a'.repeat(63)}`, true], + [`a/${'a'.repeat(64)}`, false], + ])(`isValidAnnotationKey %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidAnnotationKey(value)).toBe( + matches, + ); + }); + + it.each([ + [7, false], + [null, false], + ['', true], + ['a', true], + ['/'.repeat(6000), true], + ])(`isValidAnnotationValue %p ? %p`, (value, matches) => { + expect(KubernetesValidatorFunctions.isValidAnnotationValue(value)).toBe( + matches, + ); + }); +}); diff --git a/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts new file mode 100644 index 0000000000..fa938f5fcb --- /dev/null +++ b/packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; + +/** + * Contains validation functions that match the Kubernetes spec, usable to + * build a catalog that is compatible with those rule sets. + * + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + */ +export class KubernetesValidatorFunctions { + static isValidApiVersion(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + n => n.length >= 1 && n.length <= 63 && /^[a-z0-9A-Z]+$/.test(n), + ); + } + + static isValidKind(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-zA-Z][a-z0-9A-Z]*$/.test(value) + ); + } + + static isValidObjectName(value: any): boolean { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= 63 && + /^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value) + ); + } + + static isValidNamespace(value: any): boolean { + return CommonValidatorFunctions.isValidDnsLabel(value); + } + + static isValidLabelKey(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + KubernetesValidatorFunctions.isValidObjectName, + ); + } + + static isValidLabelValue(value: any): boolean { + return ( + value === '' || KubernetesValidatorFunctions.isValidObjectName(value) + ); + } + + static isValidAnnotationKey(value: any): boolean { + return CommonValidatorFunctions.isValidPrefixAndOrSuffix( + value, + '/', + CommonValidatorFunctions.isValidDnsSubdomain, + KubernetesValidatorFunctions.isValidObjectName, + ); + } + + static isValidAnnotationValue(value: any): boolean { + return typeof value === 'string'; + } +} diff --git a/packages/catalog-model/src/validation/index.ts b/packages/catalog-model/src/validation/index.ts new file mode 100644 index 0000000000..d679a5323c --- /dev/null +++ b/packages/catalog-model/src/validation/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CommonValidatorFunctions } from './CommonValidatorFunctions'; +export { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +export { makeValidator } from './makeValidator'; +export type { Validators } from './types'; diff --git a/packages/catalog-model/src/validation/makeValidator.ts b/packages/catalog-model/src/validation/makeValidator.ts new file mode 100644 index 0000000000..7ca01365e0 --- /dev/null +++ b/packages/catalog-model/src/validation/makeValidator.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CommonValidatorFunctions } from './CommonValidatorFunctions'; +import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions'; +import { Validators } from './types'; + +const defaultValidators: Validators = { + isValidApiVersion: KubernetesValidatorFunctions.isValidApiVersion, + isValidKind: KubernetesValidatorFunctions.isValidKind, + isValidEntityName: KubernetesValidatorFunctions.isValidObjectName, + isValidNamespace: KubernetesValidatorFunctions.isValidNamespace, + normalizeEntityName: CommonValidatorFunctions.normalizeToLowercaseAlphanum, + isValidLabelKey: KubernetesValidatorFunctions.isValidLabelKey, + isValidLabelValue: KubernetesValidatorFunctions.isValidLabelValue, + isValidAnnotationKey: KubernetesValidatorFunctions.isValidAnnotationKey, + isValidAnnotationValue: KubernetesValidatorFunctions.isValidAnnotationValue, +}; + +export function makeValidator(overrides: Partial = {}): Validators { + return { + ...defaultValidators, + ...overrides, + }; +} diff --git a/packages/catalog-model/src/validation/types.ts b/packages/catalog-model/src/validation/types.ts new file mode 100644 index 0000000000..81209bfb75 --- /dev/null +++ b/packages/catalog-model/src/validation/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type Validators = { + isValidApiVersion(value: any): boolean; + isValidKind(value: any): boolean; + isValidEntityName(value: any): boolean; + isValidNamespace(value: any): boolean; + normalizeEntityName(value: string): string; + isValidLabelKey(value: any): boolean; + isValidLabelValue(value: any): boolean; + isValidAnnotationKey(value: any): boolean; + isValidAnnotationValue(value: any): boolean; +}; diff --git a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts index b2e1dc0d32..6700f5748e 100644 --- a/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts +++ b/plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts @@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise { return knex.schema.createTable('location_update_log', table => { table.uuid('id').primary(); table.enum('status', ['success', 'fail']).notNullable(); - table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable(); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable(); table.string('message'); table .uuid('location_id') diff --git a/plugins/catalog-backend/src/ingestion/IngestionModels.ts b/plugins/catalog-backend/src/ingestion/IngestionModels.ts new file mode 100644 index 0000000000..616f1b49ae --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/IngestionModels.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model'; +import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types'; +import { LocationReader, LocationReaders } from './source'; +import { IngestionModel } from './types'; +import { DescriptorParsers } from './descriptor'; + +export class IngestionModels implements IngestionModel { + private readonly reader: LocationReader; + private readonly parser: DescriptorParser; + private readonly entityPolicy: EntityPolicy; + + static default(): IngestionModel { + return new IngestionModels( + new LocationReaders(), + new DescriptorParsers(), + new EntityPolicies(), + ); + } + + constructor( + reader: LocationReader, + parser: DescriptorParser, + entityPolicy: EntityPolicy, + ) { + this.reader = reader; + this.parser = parser; + this.entityPolicy = entityPolicy; + } + + async readLocation(type: string, target: string): Promise { + const buffer = await this.reader.tryRead(type, target); + if (!buffer) { + throw new Error(`No reader could handle location ${type} ${target}`); + } + + const items = await this.parser.tryParse(buffer); + if (!items) { + throw new Error(`No parser could handle location ${type} ${target}`); + } + + const result: ReaderOutput[] = []; + for (const item of items) { + if (item.type === 'error') { + result.push(item); + } else { + try { + const output = await this.entityPolicy.apply(item.data); + result.push({ type: 'data', data: output }); + } catch (e) { + result.push({ type: 'error', error: e }); + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts new file mode 100644 index 0000000000..ed05855109 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DescriptorParser, ReaderOutput } from './parsers/types'; +import { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; + +/** + * Parses raw descriptor data (e.g. from a file or stream) into entities. + */ +export class DescriptorParsers implements DescriptorParser { + private readonly parsers: DescriptorParser[]; + + static defaultParsers(): DescriptorParser[] { + return [new YamlDescriptorParser()]; + } + + constructor( + parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(), + ) { + this.parsers = parsers; + } + + async tryParse(data: Buffer): Promise { + for (const parser of this.parsers) { + const result = await parser.tryParse(data); + if (result) { + return result; + } + } + throw new Error(`Unsupported descriptor format`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/index.ts b/plugins/catalog-backend/src/ingestion/descriptor/index.ts new file mode 100644 index 0000000000..1529c78afc --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { DescriptorParsers } from './DescriptorParsers'; +export { YamlDescriptorParser } from './parsers/YamlDescriptorParser'; diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts new file mode 100644 index 0000000000..3f2e9e3c5c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import yaml from 'yaml'; +import { DescriptorParser, ReaderOutput } from './types'; + +/** + * Parses descriptors on YAML format + */ +export class YamlDescriptorParser implements DescriptorParser { + async tryParse(data: Buffer): Promise { + // TODO(freben): Should perhaps first do format detection, so the parse + // failure can be emitted as a proper error instead of just as if we + // weren't handling the format at all. + let documents; + try { + documents = yaml.parseAllDocuments(data.toString('utf8')); + } catch (e) { + return undefined; + } + + const result: ReaderOutput[] = []; + + for (const document of documents) { + if (document.contents) { + if (document.errors?.length) { + result.push({ + type: 'error', + error: new Error(`Malformed YAML document, ${document.errors[0]}`), + }); + } else { + const json = document.toJSON(); + if (typeof json !== 'object' || Array.isArray(json)) { + result.push({ + type: 'error', + error: new Error(`Malformed descriptor, expected object at root`), + }); + } else { + result.push({ + type: 'data', + data: json as Entity, + }); + } + } + } + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts new file mode 100644 index 0000000000..baef5dc3df --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; + +export type ReaderOutput = + | { type: 'error'; error: Error } + | { type: 'data'; data: Entity }; + +/** + * Parses raw descriptor data (e.g. from a file) into entities. + */ +export type DescriptorParser = { + /** + * Try to parse some raw data into an entity. + * + * Note that this is only the low level operation of parsing the raw file + * format, e.g. reading JSON or YAML or similar and emitting as structured + * but unvalidated data. The actual validation is performed by EntityPolicy + * and KindParser. + * + * @param data Raw descriptor data + * @returns A list of raw unvalidated entities / errors, or undefined if the + * given data is not meant to be handled by this parser + * @throws An Error if the format was handled and found to not be properly + * formed + */ + tryParse(data: Buffer): Promise; +}; diff --git a/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts new file mode 100644 index 0000000000..a670f309ad --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/LocationReaders.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileLocationReader } from './readers/FileLocationReader'; +import { GitHubLocationReader } from './readers/GitHubLocationReader'; +import { LocationReader } from './readers/types'; + +export class LocationReaders implements LocationReader { + private readonly readers: LocationReader[]; + + static defaultReaders(): LocationReader[] { + return [new FileLocationReader(), new GitHubLocationReader()]; + } + + constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) { + this.readers = readers; + } + + async tryRead(type: string, target: string): Promise { + for (const reader of this.readers) { + const result = await reader.tryRead(type, target); + if (result) { + return result; + } + } + throw new Error(`Could not read unknown location "${type}", "${target}"`); + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/index.ts b/plugins/catalog-backend/src/ingestion/source/index.ts new file mode 100644 index 0000000000..3ed1063878 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { LocationReaders } from './LocationReaders'; +export { FileLocationReader } from './readers/FileLocationReader'; +export { GitHubLocationReader } from './readers/GitHubLocationReader'; +export { LocationReader } from './readers/types'; diff --git a/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts new file mode 100644 index 0000000000..0c64aebf14 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { LocationReader } from './types'; + +/** + * Reads a file from the local file system. + */ +export class FileLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'file') { + return undefined; + } + + try { + return await fs.readFile(target); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts new file mode 100644 index 0000000000..1f0f6ed539 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('node-fetch'); + +import fetch from 'node-fetch'; +import { GitHubLocationReader } from './GitHubLocationReader'; + +const { Response } = jest.requireActual('node-fetch'); + +describe('Unit: GitHubLocationReader', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('fetches the file and parses it correctly', async () => { + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + const reader = new GitHubLocationReader(); + const buffer = await reader.tryRead( + 'github', + 'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml', + ); + + expect(buffer?.toString('utf8')).toBe('hello'); + }); + + it('changes the url to point to https://raw.githubusercontent.com', async () => { + const gitHubUrl = `https://github.com`; + const project = `spotify/backstage`; + const folderPath = `master/plugins/catalog-backend/fixtures`; + const componentFilename = `one_component.yaml`; + const rawGitHubUrl = `https://raw.githubusercontent.com`; + + const reader = new GitHubLocationReader(); + (fetch as any).mockResolvedValueOnce(new Response('hello')); + + await reader.tryRead( + 'github', + `${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`, + ); + + expect(fetch).toHaveBeenCalledWith( + `${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`, + ); + }); + + describe('rejects wrong urls', () => { + const reader = new GitHubLocationReader(); + + it.each([ + ['http://example.com/one_component.yaml'], + ['http://github.com/one_component.yaml'], + ['http://github.com/PROJECT/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.yaml'], + ['http://github.com/PROJECT/REPO/one_component.json'], + ])( + '%p', + async (url: string) => + await expect(reader.tryRead('github', url)).rejects.toThrow(/url/), + ); + }); +}); + +describe('Integration: GitHubLocationSource', () => { + beforeAll(() => { + (fetch as any).mockImplementation(jest.requireActual('node-fetch')); + }); + + it('fetches the fixture from backstage repo', async () => { + const PERMANENT_LINK = + 'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml'; + + const reader = new GitHubLocationReader(); + const result = await reader.tryRead('github', PERMANENT_LINK); + + expect(result?.toString('utf8')).toContain('component3'); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts new file mode 100644 index 0000000000..bd330e28b4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'node-fetch'; +import { URL } from 'url'; +import { LocationReader } from './types'; + +/** + * Reads a file whose target is a GitHub URL. + * + * Uses raw.githubusercontent.com for now, but this will probably change in the + * future when token auth is implemented. + */ +export class GitHubLocationReader implements LocationReader { + async tryRead(type: string, target: string): Promise { + if (type !== 'github') { + return undefined; + } + + const url = this.buildRawUrl(target); + try { + return await fetch(url.toString()).then(x => x.buffer()); + } catch (e) { + throw new Error(`Unable to read "${target}", ${e}`); + } + } + + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/source/readers/types.ts b/plugins/catalog-backend/src/ingestion/source/readers/types.ts new file mode 100644 index 0000000000..37c7b46885 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/source/readers/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type LocationReader = { + /** + * Reads the contents of a single location. + * + * @param type The type of location to read + * @param target The location target (type-specific) + * @returns The target contents, as a raw Buffer, or undefined if this type + * was not meant to be consumed by this reader + * @throws An error if the type was meant for this reader, but could not be + * read + */ + tryRead(type: string, target: string): Promise; +}; diff --git a/yarn.lock b/yarn.lock index 6390797c12..74b9e451b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17169,11 +17169,6 @@ rc@^1.0.1, rc@^1.1.6, rc@^1.2.7, rc@^1.2.8: minimist "^1.2.0" strip-json-comments "~2.0.1" -react-addons-text-content@0.0.4: - version "0.0.4" - resolved "https://registry.npmjs.org/react-addons-text-content/-/react-addons-text-content-0.0.4.tgz#d2e259fdc951d1d8906c08902002108dce8792e5" - integrity sha1-0uJZ/clR0diQbAiQIAIQjc6HkuU= - react-beautiful-dnd@11.0.3: version "11.0.3" resolved "https://registry.npmjs.org/react-beautiful-dnd/-/react-beautiful-dnd-11.0.3.tgz#5678bb3e725d8b56cb7cf57f56e952105fc4f2af"