paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

Merge pull request #1047 from spotify/freben/catalog-model

Browse Source 
Move the model into a plugin-catalog-model for sharing outside the backend
This commit is contained in:
Fredrik Adelöw
2020-05-29 07:02:12 +02:00
committed by GitHub
parent 828e6675c8 06879f5542
commit 273f009297
62 changed files with 1596 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/package.json
+5 -3
View File
@@ -10,7 +10,7 @@
},
"scripts": {
"build": "tsc",
"start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess nodemon",
"start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"nodemon -r esm\\\"",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"clean": "backstage-cli clean",
@@ -18,13 +18,15 @@
},
"dependencies": {
"@backstage/backend-common": "^0.1.1-alpha.6",
"@backstage/catalog-model": "^0.1.1-alpha.6",
"@backstage/plugin-auth-backend": "^0.1.1-alpha.6",
"@backstage/plugin-catalog-backend": "^0.1.1-alpha.6",
"@backstage/plugin-sentry-backend": "^0.1.1-alpha.6",
"@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6",
"@backstage/plugin-identity-backend": "^0.1.1-alpha.6",
"@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6",
"@backstage/plugin-sentry-backend": "^0.1.1-alpha.6",
"compression": "^1.7.4",
"cors": "^2.8.5",
"esm": "^3.2.25",
"express": "^4.17.1",
"helmet": "^3.22.0",
"knex": "^0.21.1",
packages/backend/src/plugins/catalog.ts
+10 -4
View File
@@ -21,22 +21,28 @@ import {
DatabaseManager,
DescriptorParsers,
LocationReaders,
IngestionModels,
runPeriodically,
} from '@backstage/plugin-catalog-backend';
import { PluginEnvironment } from '../types';
import { EntityPolicies } from '@backstage/catalog-model';
export default async function ({ logger, database }: PluginEnvironment) {
const reader = LocationReaders.create();
const parser = DescriptorParsers.create();
const policy = new EntityPolicies();
const ingestion = new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
const db = await DatabaseManager.createDatabase(database, logger);
runPeriodically(
() => DatabaseManager.refreshLocations(db, reader, parser, logger),
() => DatabaseManager.refreshLocations(db, ingestion, policy, logger),
10000,
);
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db, reader);
const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion);
return await createRouter({ entitiesCatalog, locationsCatalog, logger });
}
packages/catalog-model/.eslintrc.js
+3
View File
@@ -0,0 +1,3 @@
module.exports = {
extends: [require.resolve('@backstage/cli/config/eslint')],
};
packages/catalog-model/README.md
+12
View File
@@ -0,0 +1,12 @@
# Catalog Model
Contains the core model types and validators/policies used by the Backstage catalog functionality.
This package will be imported both by the frontend and backend parts of the catalog,
as well as by others that want to consume catalog data.
## Links
- (Default frontend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog]
- (Default backend part of the catalog)[https://github.com/spotify/backstage/tree/master/plugins/catalog-backend]
- (The Backstage homepage)[https://backstage.io]
packages/catalog-model/package.json
+34
View File
@@ -0,0 +1,34 @@
{
"name": "@backstage/catalog-model",
"version": "0.1.1-alpha.6",
"main": "dist/index.esm.js",
"main:src": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": true,
"publishConfig": {
"access": "public"
},
"scripts": {
"build": "backstage-cli plugin:build",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"prepack": "backstage-cli prepack",
"postpack": "backstage-cli postpack",
"clean": "backstage-cli clean"
},
"dependencies": {
"lodash": "^4.17.15",
"yup": "^0.28.5"
},
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.6",
"@types/jest": "^25.2.2",
"@types/lodash": "^4.14.151",
"@types/yup": "^0.28.2",
"yaml": "^1.9.2"
},
"files": [
"dist/**/*.{js,d.ts}"
]
}
packages/catalog-model/src/EntityPolicies.ts
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
FieldFormatEntityPolicy,
NoForeignRootFieldsEntityPolicy,
ReservedFieldsEntityPolicy,
SchemaValidEntityPolicy,
} from './entity';
import { ComponentV1beta1Policy } from './kinds';
import { EntityPolicy } from './types';
// Helper that requires that all of a set of policies can be successfully
// applied
class AllEntityPolicies implements EntityPolicy {
constructor(private readonly policies: EntityPolicy[]) {}
async enforce(entity: Entity): Promise<Entity> {
let result = entity;
for (const policy of this.policies) {
result = await policy.enforce(entity);
}
return result;
}
}
// Helper that requires that at least one of a set of policies can be
// successfully applied
class AnyEntityPolicy implements EntityPolicy {
constructor(private readonly policies: EntityPolicy[]) {}
async enforce(entity: Entity): Promise<Entity> {
for (const policy of this.policies) {
try {
return await policy.enforce(entity);
} catch {
continue;
}
}
throw new Error(`The entity did not match any known policy`);
}
}
export class EntityPolicies implements EntityPolicy {
private readonly policy: EntityPolicy;
static defaultPolicies(): EntityPolicy {
return EntityPolicies.allOf([
EntityPolicies.allOf([
new SchemaValidEntityPolicy(),
new NoForeignRootFieldsEntityPolicy(),
new FieldFormatEntityPolicy(),
new ReservedFieldsEntityPolicy(),
]),
EntityPolicies.anyOf([new ComponentV1beta1Policy()]),
]);
}
static allOf(policies: EntityPolicy[]): EntityPolicy {
return new AllEntityPolicies(policies);
}
static anyOf(policies: EntityPolicy[]): EntityPolicy {
return new AnyEntityPolicy(policies);
}
constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) {
this.policy = policy;
}
enforce(entity: Entity): Promise<Entity> {
return this.policy.enforce(entity);
}
}
packages/catalog-model/src/entity/Entity.ts
+108
View File
@@ -0,0 +1,108 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* The format envelope that's common to all versions/kinds of entity.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
*/
export type Entity = {
/**
* The version of specification format for this particular entity that
* this is written against.
*/
apiVersion: string;
/**
* The high level entity type being described.
*/
kind: string;
/**
* Optional metadata related to the entity.
*/
metadata?: EntityMeta;
/**
* The specification data describing the entity itself.
*/
spec?: object;
};
/**
* Metadata fields common to all versions/kinds of entity.
*
* @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
*/
export type EntityMeta = {
/**
* A globally unique ID for the entity.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, but the server is free to reject requests
* that do so in such a way that it breaks semantics.
*/
uid?: string;
/**
* An opaque string that changes for each update operation to any part of
* the entity, including metadata.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, and the server will then reject the
* operation if it does not match the current stored value.
*/
etag?: string;
/**
* A positive nonzero number that indicates the current generation of data
* for this entity; the value is incremented each time the spec changes.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations.
*/
generation?: number;
/**
* The name of the entity.
*
* Must be uniqe within the catalog at any given point in time, for any
* given namespace, for any given kind.
*/
name?: string;
/**
* The namespace that the entity belongs to.
*/
namespace?: string;
/**
* Key/value pairs of identifying information attached to the entity.
*/
labels?: Record<string, string>;
/**
* Key/value pairs of non-identifying auxiliary information attached to the
* entity.
*/
annotations?: Record<string, string>;
};
plugins/catalog-backend/src/validation/index.ts → packages/catalog-model/src/entity/index.ts
+2 -4
View File
@@ -14,7 +14,5 @@
* limitations under the License.
*/
export * from './CommonValidatorFunctions';
export * from './KubernetesValidatorFunctions';
export * from './makeValidator';
export * from './types';
export type { Entity, EntityMeta } from './Entity';
export * from './policies';
packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.test.ts
+105
View File
@@ -0,0 +1,105 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy';
describe('FieldFormatEntityPolicy', () => {
let data: any;
let policy: FieldFormatEntityPolicy;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
policy = new FieldFormatEntityPolicy();
});
it('works for the happy path', async () => {
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad apiVersion', async () => {
data.apiVersion = 7;
await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/);
data.apiVersion = 'a#b';
await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/);
});
it('rejects bad kind', async () => {
data.kind = 7;
await expect(policy.enforce(data)).rejects.toThrow(/kind/);
data.kind = 'a#b';
await expect(policy.enforce(data)).rejects.toThrow(/kind/);
});
it('handles missing metadata gracefully', async () => {
delete data.medatata;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('handles missing spec gracefully', async () => {
delete data.spec;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad name', async () => {
data.metadata.name = 7;
await expect(policy.enforce(data)).rejects.toThrow(/name.*7/);
data.metadata.name = 'a'.repeat(1000);
await expect(policy.enforce(data)).rejects.toThrow(/name.*aaaa/);
});
it('rejects bad namespace', async () => {
data.metadata.namespace = 7;
await expect(policy.enforce(data)).rejects.toThrow(/namespace.*7/);
data.metadata.namespace = 'a'.repeat(1000);
await expect(policy.enforce(data)).rejects.toThrow(/namespace.*aaaa/);
});
it('rejects bad label key', async () => {
data.metadata.labels['a#b'] = 'value';
await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i);
});
it('rejects bad label value', async () => {
data.metadata.labels.a = 'a#b';
await expect(policy.enforce(data)).rejects.toThrow(/label.*a#b/i);
});
it('rejects bad annotation key', async () => {
data.metadata.annotations['a#b'] = 'value';
await expect(policy.enforce(data)).rejects.toThrow(/annotation.*a#b/i);
});
it('rejects bad annotation value', async () => {
data.metadata.annotations.a = 7;
await expect(policy.enforce(data)).rejects.toThrow(/annotation.*7/i);
});
});
packages/catalog-model/src/entity/policies/FieldFormatEntityPolicy.ts
+91
View File
@@ -0,0 +1,91 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicy } from '../../types';
import { makeValidator, Validators } from '../../validation';
import { Entity } from '../Entity';
/**
* Ensures that the format of individual fields of the entity envelope
* is valid.
*
* This does not take into account machine generated fields such as uid, etag
* and generation.
*/
export class FieldFormatEntityPolicy implements EntityPolicy {
private readonly validators: Validators;
constructor(validators: Validators = makeValidator()) {
this.validators = validators;
}
async enforce(entity: Entity): Promise<Entity> {
function require(
field: string,
value: any,
validator: (value: any) => boolean,
) {
if (value === undefined || value === null) {
throw new Error(`${field} must have a value`);
}
let isValid: boolean;
try {
isValid = validator(value);
} catch (e) {
throw new Error(`${field} could not be validated, ${e}`);
}
if (!isValid) {
throw new Error(`${field} "${value}" is not valid`);
}
}
function optional(
field: string,
value: any,
validator: (value: any) => boolean,
) {
return value === undefined || require(field, value, validator);
}
require('apiVersion', entity.apiVersion, this.validators.isValidApiVersion);
require('kind', entity.kind, this.validators.isValidKind);
optional(
'metadata.name',
entity.metadata?.name,
this.validators.isValidEntityName,
);
optional(
'metadata.namespace',
entity.metadata?.namespace,
this.validators.isValidNamespace,
);
for (const [k, v] of Object.entries(entity.metadata?.labels ?? [])) {
require(`labels.${k}`, k, this.validators.isValidLabelKey);
require(`labels.${k}`, v, this.validators.isValidLabelValue);
}
for (const [k, v] of Object.entries(entity.metadata?.annotations ?? [])) {
require(`annotations.${k}`, k, this.validators.isValidAnnotationKey);
require(`annotations.${k}`, v, this.validators.isValidAnnotationValue);
}
return entity;
}
}
packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.test.ts
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy';
describe('NoForeignRootFieldsEntityPolicy', () => {
let data: any;
let policy: NoForeignRootFieldsEntityPolicy;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
policy = new NoForeignRootFieldsEntityPolicy();
});
it('works for the happy path', async () => {
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects unknown root fields', async () => {
data.spec2 = {};
await expect(policy.enforce(data)).rejects.toThrow(/spec2/i);
});
});
packages/catalog-model/src/entity/policies/NoForeignRootFieldsEntityPolicy.ts
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicy } from '../../types';
import { Entity } from '../Entity';
const defaultKnownFields = ['apiVersion', 'kind', 'metadata', 'spec'];
/**
* Ensures that there are no foreign root fields in the entity.
*/
export class NoForeignRootFieldsEntityPolicy implements EntityPolicy {
private readonly knownFields: string[];
constructor(knownFields: string[] = defaultKnownFields) {
this.knownFields = knownFields;
}
async enforce(entity: Entity): Promise<Entity> {
for (const field of Object.keys(entity)) {
if (!this.knownFields.includes(field)) {
throw new Error(`Unknown field ${field}`);
}
}
return entity;
}
}
packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.test.ts
+64
View File
@@ -0,0 +1,64 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy';
describe('ReservedFieldsEntityPolicy', () => {
let data: any;
let policy: ReservedFieldsEntityPolicy;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
policy = new ReservedFieldsEntityPolicy();
});
it('works for the happy path', async () => {
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects reserved keys in the spec root', async () => {
data.spec.apiVersion = 'a/b';
await expect(policy.enforce(data)).rejects.toThrow(/spec.*apiVersion/i);
});
it('rejects reserved keys in labels', async () => {
data.metadata.labels.apiVersion = 'a';
await expect(policy.enforce(data)).rejects.toThrow(/label.*apiVersion/i);
});
it('rejects reserved keys in annotations', async () => {
data.metadata.annotations.apiVersion = 'a';
await expect(policy.enforce(data)).rejects.toThrow(
/annotation.*apiVersion/i,
);
});
});
packages/catalog-model/src/entity/policies/ReservedFieldsEntityPolicy.ts
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicy } from '../../types';
import { Entity } from '../Entity';
const DEFAULT_RESERVED_ENTITY_FIELDS = [
'apiVersion',
'kind',
'uid',
'etag',
'generation',
'name',
'namespace',
'labels',
'annotations',
'spec',
];
/**
* Ensures that fields are not given certain reserved names.
*/
export class ReservedFieldsEntityPolicy implements EntityPolicy {
private readonly reservedFields: string[];
constructor(fields?: string[]) {
this.reservedFields = [
...(fields ?? []),
...DEFAULT_RESERVED_ENTITY_FIELDS,
];
}
async enforce(entity: Entity): Promise<Entity> {
for (const field of this.reservedFields) {
if (entity.spec?.hasOwnProperty(field)) {
throw new Error(
`The spec may not contain the field ${field}, because it has reserved meaning`,
);
}
if (entity.metadata?.labels?.hasOwnProperty(field)) {
throw new Error(
`A label may not have the field ${field}, because it has reserved meaning`,
);
}
if (entity.metadata?.annotations?.hasOwnProperty(field)) {
throw new Error(
`An annotation may not have the field ${field}, because it has reserved meaning`,
);
}
}
return entity;
}
}
packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts
+176
View File
@@ -0,0 +1,176 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { Entity } from '../Entity';
import { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy';
describe('SchemaValidEntityPolicy', () => {
let data: any;
let policy: SchemaValidEntityPolicy;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
policy = new SchemaValidEntityPolicy();
});
it('works for the happy path', async () => {
await expect(policy.enforce(data)).resolves.toBe(data);
});
//
// apiVersion and kind
//
it('rejects wrong root type', async () => {
await expect(policy.enforce((7 as unknown) as Entity)).rejects.toThrow(
/object/,
);
});
it('rejects missing apiVersion', async () => {
delete data.apiVersion;
await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/);
});
it('rejects bad apiVersion type', async () => {
data.apiVersion = 7;
await expect(policy.enforce(data)).rejects.toThrow(/apiVersion/);
});
it('rejects missing kind', async () => {
delete data.kind;
await expect(policy.enforce(data)).rejects.toThrow(/kind/);
});
it('rejects bad kind type', async () => {
data.kind = 7;
await expect(policy.enforce(data)).rejects.toThrow(/kind/);
});
//
// metadata
//
it('accepts missing metadata', async () => {
delete data.metadata;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad metadata type', async () => {
data.metadata = 7;
await expect(policy.enforce(data)).rejects.toThrow(/metadata/);
});
it('accepts missing uid', async () => {
delete data.metadata.uid;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad uid type', async () => {
data.metadata.uid = 7;
await expect(policy.enforce(data)).rejects.toThrow(/uid/);
});
it('accepts missing etag', async () => {
delete data.metadata.etag;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad etag type', async () => {
data.metadata.etag = 7;
await expect(policy.enforce(data)).rejects.toThrow(/etag/);
});
it('accepts missing generation', async () => {
delete data.metadata.generation;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad generation type', async () => {
data.metadata.generation = 'a';
await expect(policy.enforce(data)).rejects.toThrow(/generation/);
});
it('accepts missing name', async () => {
delete data.metadata.name;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad name type', async () => {
data.metadata.name = 7;
await expect(policy.enforce(data)).rejects.toThrow(/name/);
});
it('accepts missing namespace', async () => {
delete data.metadata.namespace;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad namespace type', async () => {
data.metadata.namespace = 7;
await expect(policy.enforce(data)).rejects.toThrow(/namespace/);
});
it('accepts missing labels', async () => {
delete data.metadata.labels;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad labels type', async () => {
data.metadata.labels = 7;
await expect(policy.enforce(data)).rejects.toThrow(/labels/);
});
it('accepts missing annotations', async () => {
delete data.metadata.annotations;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects bad annotations type', async () => {
data.metadata.annotations = 7;
await expect(policy.enforce(data)).rejects.toThrow(/annotations/);
});
//
// spec
//
it('accepts missing spec', async () => {
delete data.spec;
await expect(policy.enforce(data)).resolves.toBe(data);
});
it('rejects non-object spec', async () => {
data.spec = 7;
await expect(policy.enforce(data)).rejects.toThrow(/spec/);
});
});
packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.ts
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { EntityPolicy } from '../../types';
import { Entity } from '../Entity';
const DEFAULT_ENTITY_SCHEMA = yup.object({
apiVersion: yup.string().required(),
kind: yup.string().required(),
metadata: yup
.object({
uid: yup
.string()
.notRequired()
.test(
'metadata.uid',
'The uid must not be empty',
value => value === undefined || value.length > 0,
),
etag: yup
.string()
.notRequired()
.test(
'metadata.etag',
'The etag must not be empty',
value => value === undefined || value.length > 0,
),
generation: yup
.number()
.notRequired()
.test(
'metadata.generation',
'The generation must be an integer greater than zero',
value => value === undefined || (value === (value | 0) && value > 0),
),
name: yup.string().notRequired(),
namespace: yup.string().notRequired(),
labels: yup.object<Record<string, string>>().notRequired(),
annotations: yup.object<Record<string, string>>().notRequired(),
})
.notRequired(),
spec: yup.object({}).notRequired(),
});
/**
* Ensures that the entity spec is valid according to a schema.
*
* This should be the first policy in the list, to ensure that other downstream
* policies can work with a structure that is at least valid in therms of the
* typescript type.
*/
export class SchemaValidEntityPolicy implements EntityPolicy {
private readonly schema: yup.Schema<Entity>;
constructor(schema: yup.Schema<Entity> = DEFAULT_ENTITY_SCHEMA) {
this.schema = schema;
}
async enforce(entity: Entity): Promise<Entity> {
try {
return await this.schema.validate(entity, { strict: true });
} catch (e) {
throw new Error(`Malformed envelope, ${e}`);
}
}
}
packages/catalog-model/src/entity/policies/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { FieldFormatEntityPolicy } from './FieldFormatEntityPolicy';
export { NoForeignRootFieldsEntityPolicy } from './NoForeignRootFieldsEntityPolicy';
export { ReservedFieldsEntityPolicy } from './ReservedFieldsEntityPolicy';
export { SchemaValidEntityPolicy } from './SchemaValidEntityPolicy';
packages/catalog-model/src/index.ts
+21
View File
@@ -0,0 +1,21 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './entity';
export { EntityPolicies } from './EntityPolicies';
export * from './kinds';
export type { EntityPolicy } from './types';
export * from './validation';
plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts → packages/catalog-model/src/kinds/ComponentV1beta1.ts
+16 -16
View File
@@ -15,19 +15,28 @@
*/
import * as yup from 'yup';
import { DescriptorEnvelope, KindParser, ParserError } from '../types';
import type { Entity, EntityMeta } from '../entity/Entity';
import type { EntityPolicy } from '../types';
export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope {
const API_VERSION = 'backstage.io/v1beta1';
const KIND = 'Component';
export interface ComponentV1beta1 extends Entity {
apiVersion: typeof API_VERSION;
kind: typeof KIND;
metadata: EntityMeta & {
name: string;
};
spec: {
type: string;
};
}
export class ComponentDescriptorV1beta1Parser implements KindParser {
export class ComponentV1beta1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<ComponentDescriptorV1beta1>>({
this.schema = yup.object<Partial<ComponentV1beta1>>({
metadata: yup
.object({
name: yup.string().required(),
@@ -41,23 +50,14 @@ export class ComponentDescriptorV1beta1Parser implements KindParser {
});
}
async tryParse(
envelope: DescriptorEnvelope,
): Promise<DescriptorEnvelope | undefined> {
async enforce(envelope: Entity): Promise<Entity> {
if (
envelope.apiVersion !== 'backstage.io/v1beta1' ||
envelope.kind !== 'Component'
) {
return undefined;
throw new Error('Unsupported apiVersion / kind');
}
try {
return await this.schema.validate(envelope, { strict: true });
} catch (e) {
throw new ParserError(
`Malformed component, ${e}`,
envelope.metadata?.name,
);
}
return await this.schema.validate(envelope, { strict: true });
}
}
packages/catalog-model/src/kinds/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { ComponentV1beta1 } from './ComponentV1beta1';
export { ComponentV1beta1Policy } from './ComponentV1beta1';
export { ComponentV1beta1 as Component };
export { ComponentV1beta1 };
packages/catalog-model/src/types.ts
+32
View File
@@ -0,0 +1,32 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Entity } from './entity/Entity';
/**
* A policy for validation or mutation to be applied to entities as they are
* entering the system.
*/
export type EntityPolicy = {
/**
* Applies validation or mutation on an entity.
*
* @param entity The entity, as validated/mutated so far in the policy tree
* @returns The incoming entity, or a mutated version of the same
* @throws An error if the entity should be rejected
*/
enforce(entity: Entity): Promise<Entity>;
};
plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts → packages/catalog-model/src/validation/CommonValidatorFunctions.test.ts
View File
plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts → packages/catalog-model/src/validation/CommonValidatorFunctions.ts
View File
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts → packages/catalog-model/src/validation/KubernetesValidatorFunctions.test.ts
View File
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts → packages/catalog-model/src/validation/KubernetesValidatorFunctions.ts
View File
packages/catalog-model/src/validation/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { CommonValidatorFunctions } from './CommonValidatorFunctions';
export { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions';
export { makeValidator } from './makeValidator';
export type { Validators } from './types';
plugins/catalog-backend/src/validation/makeValidator.ts → packages/catalog-model/src/validation/makeValidator.ts
View File
plugins/catalog-backend/src/validation/types.ts → packages/catalog-model/src/validation/types.ts
View File
plugins/catalog-backend/package.json
+3 -2
View File
@@ -16,8 +16,7 @@
},
"dependencies": {
"@backstage/backend-common": "^0.1.1-alpha.6",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@backstage/catalog-model": "^0.1.1-alpha.6",
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
@@ -38,6 +37,8 @@
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.6",
"@types/lodash": "^4.14.151",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
"@types/yup": "^0.28.2",
"jest-fetch-mock": "^3.0.3",
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+4 -4
View File
@@ -14,21 +14,21 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { Database } from '../database';
import { DescriptorEnvelope } from '../ingestion/types';
import { EntitiesCatalog, EntityFilters } from './types';
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
constructor(private readonly database: Database) {}
async entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]> {
async entities(filters?: EntityFilters): Promise<Entity[]> {
const items = await this.database.transaction(tx =>
this.database.entities(tx, filters),
);
return items.map(i => i.entity);
}
async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
async entityByUid(uid: string): Promise<Entity | undefined> {
const matches = await this.database.transaction(tx =>
this.database.entities(tx, [{ key: 'uid', values: [uid] }]),
);
@@ -40,7 +40,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
kind: string,
name: string,
namespace: string | undefined,
): Promise<DescriptorEnvelope | undefined> {
): Promise<Entity | undefined> {
const matches = await this.database.transaction(tx =>
this.database.entities(tx, [
{ key: 'kind', values: [kind] },
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts
+21 -19
View File
@@ -13,13 +13,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import knex from 'knex';
import path from 'path';
import { Database } from '../database';
import { ReaderOutput } from '../ingestion/types';
import { getVoidLogger } from '@backstage/backend-common';
import { IngestionModel } from '../ingestion/types';
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
class MockIngestionModel implements IngestionModel {
readLocation = jest.fn(async (type: string, target: string) => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return [{ type: 'data', data: {} as Entity } as const];
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
});
}
describe('DatabaseLocationsCatalog', () => {
const database = knex({
@@ -32,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => {
});
let db: Database;
let catalog: DatabaseLocationsCatalog;
const mockLocationReader = {
read: async (type: string, target: string): Promise<ReaderOutput[]> => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return Promise.resolve([{ type: 'data', data: {} }]);
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
},
};
let ingestionModel: IngestionModel;
beforeEach(async () => {
await database.migrate.latest({
@@ -53,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => {
loadExtensions: ['.ts'],
});
db = new Database(database, getVoidLogger());
catalog = new DatabaseLocationsCatalog(db, mockLocationReader);
ingestionModel = new MockIngestionModel();
catalog = new DatabaseLocationsCatalog(db, ingestionModel);
});
it('resolves to location with id', async () => {
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts
+11 -3
View File
@@ -15,17 +15,25 @@
*/
import { Database } from '../database';
import { IngestionModel } from '../ingestion/types';
import { AddLocation, Location, LocationsCatalog } from './types';
import { LocationReader } from '../ingestion';
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(
private readonly database: Database,
private readonly reader: LocationReader,
private readonly ingestionModel: IngestionModel,
) {}
async addLocation(location: AddLocation): Promise<Location> {
const outputs = await this.reader.read(location.type, location.target);
const outputs = await this.ingestionModel.readLocation(
location.type,
location.target,
);
if (!outputs) {
throw new Error(
`Unknown location type ${location.type} ${location.target}`,
);
}
outputs.forEach(output => {
if (output.type === 'error') {
throw new Error(
plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
+6 -6
View File
@@ -15,22 +15,22 @@
*/
import { NotFoundError } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import lodash from 'lodash';
import { DescriptorEnvelope } from '../ingestion';
import { EntitiesCatalog } from './types';
export class StaticEntitiesCatalog implements EntitiesCatalog {
private _entities: DescriptorEnvelope[];
private _entities: Entity[];
constructor(entities: DescriptorEnvelope[]) {
constructor(entities: Entity[]) {
this._entities = entities;
}
async entities(): Promise<DescriptorEnvelope[]> {
async entities(): Promise<Entity[]> {
return lodash.cloneDeep(this._entities);
}
async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
async entityByUid(uid: string): Promise<Entity | undefined> {
const item = this._entities.find(e => uid === e.metadata?.uid);
if (!item) {
throw new NotFoundError('Entity cannot be found');
@@ -42,7 +42,7 @@ export class StaticEntitiesCatalog implements EntitiesCatalog {
kind: string,
name: string,
namespace: string | undefined,
): Promise<DescriptorEnvelope | undefined> {
): Promise<Entity | undefined> {
const item = this._entities.find(
e =>
kind === e.kind &&
plugins/catalog-backend/src/catalog/types.ts
+4 -4
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion';
//
// Entities
@@ -28,13 +28,13 @@ export type EntityFilter = {
export type EntityFilters = EntityFilter[];
export type EntitiesCatalog = {
entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]>;
entityByUid(uid: string): Promise<DescriptorEnvelope | undefined>;
entities(filters?: EntityFilters): Promise<Entity[]>;
entityByUid(uid: string): Promise<Entity | undefined>;
entityByName(
kind: string,
namespace: string | undefined,
name: string,
): Promise<DescriptorEnvelope | undefined>;
): Promise<Entity | undefined>;
};
//
plugins/catalog-backend/src/database/Database.test.ts
+5 -5
View File
@@ -19,9 +19,9 @@ import {
getVoidLogger,
NotFoundError,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import Knex from 'knex';
import path from 'path';
import { DescriptorEnvelope } from '../ingestion';
import { Database } from './Database';
import {
AddDatabaseLocation,
@@ -247,8 +247,8 @@ describe('Database', () => {
describe('entities', () => {
it('can get all entities with empty filters list', async () => {
const catalog = new Database(database, getVoidLogger());
const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' };
const e2: DescriptorEnvelope = {
const e1: Entity = { apiVersion: 'a', kind: 'b' };
const e2: Entity = {
apiVersion: 'a',
kind: 'b',
spec: { c: null },
@@ -271,7 +271,7 @@ describe('Database', () => {
it('can get all specific entities for matching filters (naive case)', async () => {
const catalog = new Database(database, getVoidLogger());
const entities: DescriptorEnvelope[] = [
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b' },
{
apiVersion: 'a',
@@ -305,7 +305,7 @@ describe('Database', () => {
it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => {
const catalog = new Database(database, getVoidLogger());
const entities: DescriptorEnvelope[] = [
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b' },
{
apiVersion: 'a',
plugins/catalog-backend/src/database/Database.ts
+12 -10
View File
@@ -19,12 +19,12 @@ import {
InputError,
NotFoundError,
} from '@backstage/backend-common';
import { Entity, EntityMeta } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntityFilters } from '../catalog';
import { DescriptorEnvelope, EntityMeta } from '../ingestion';
import { buildEntitySearch } from './search';
import {
AddDatabaseLocation,
@@ -54,9 +54,7 @@ function serializeMetadata(metadata: EntityMeta | undefined): string | null {
return JSON.stringify(getStrippedMetadata(metadata));
}
function serializeSpec(
spec: DescriptorEnvelope['spec'],
): DbEntitiesRow['spec'] {
function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] {
if (!spec) {
return null;
}
@@ -66,7 +64,7 @@ function serializeSpec(
function toEntityRow(
locationId: string | undefined,
entity: DescriptorEnvelope,
entity: Entity,
): DbEntitiesRow {
return {
id: entity.metadata!.uid!,
@@ -83,7 +81,7 @@ function toEntityRow(
}
function toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: row.api_version,
kind: row.kind,
metadata: {
@@ -94,7 +92,7 @@ function toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
};
if (row.metadata) {
const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata'];
const metadata = JSON.parse(row.metadata) as Entity['metadata'];
entity.metadata = { ...entity.metadata, ...metadata };
}
@@ -127,7 +125,9 @@ function generateUid(): string {
}
function generateEtag(): string {
return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
return Buffer.from(uuidv4(), 'utf8')
.toString('base64')
.replace(/[^\w]/g, '');
}
/**
@@ -374,7 +374,9 @@ export class Database {
target,
});
return (await tx<DbLocationsRow>('locations').where({ id }).select())![0];
return (await tx<DbLocationsRow>('locations')
.where({ id })
.select())![0];
});
}
@@ -422,7 +424,7 @@ export class Database {
private async updateEntitiesSearch(
tx: Knex.Transaction<any, any>,
entityId: string,
data: DescriptorEnvelope,
data: Entity,
): Promise<void> {
try {
const entries = buildEntitySearch(entityId, data);
plugins/catalog-backend/src/database/DatabaseManager.test.ts
+42 -40
View File
@@ -15,13 +15,9 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import {
ComponentDescriptor,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
@@ -32,18 +28,18 @@ describe('DatabaseManager', () => {
const db = ({
locations: jest.fn().mockResolvedValue([]),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(),
const reader: IngestionModel = {
readLocation: jest.fn(),
};
const parser: DescriptorParser = {
parse: jest.fn(),
const policy: EntityPolicy = {
enforce: jest.fn(),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).not.toHaveBeenCalled();
expect(parser.parse).not.toHaveBeenCalled();
expect(reader.readLocation).not.toHaveBeenCalled();
expect(policy.enforce).not.toHaveBeenCalled();
});
it('can update a single location', async () => {
@@ -52,7 +48,7 @@ describe('DatabaseManager', () => {
type: 'some',
target: 'thing',
};
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
@@ -68,18 +64,20 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as Partial<Database>) as Database;
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).toHaveBeenCalledTimes(1);
expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(reader.readLocation).toHaveBeenCalledTimes(1);
expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(db.addEntity).toHaveBeenCalledTimes(1);
expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, {
locationId: '123',
@@ -108,21 +106,23 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -158,23 +158,25 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -211,19 +213,19 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(() =>
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.reject([{ type: 'error', error: new Error('test message') }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
plugins/catalog-backend/src/database/DatabaseManager.ts
+12 -20
View File
@@ -14,16 +14,12 @@
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import path from 'path';
import { Logger } from 'winston';
import {
DescriptorEnvelope,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types';
@@ -67,8 +63,8 @@ export class DatabaseManager {
public static async refreshLocations(
database: Database,
reader: LocationReader,
parser: DescriptorParser,
ingestionModel: IngestionModel,
entityPolicy: EntityPolicy,
logger: Logger,
): Promise<void> {
const locations = await database.locations();
@@ -78,7 +74,10 @@ export class DatabaseManager {
`Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
);
const readerOutput = await reader.read(location.type, location.target);
const readerOutput = await ingestionModel.readLocation(
location.type,
location.target,
);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
@@ -87,7 +86,7 @@ export class DatabaseManager {
}
try {
const entity = await parser.parse(readerItem.data);
const entity = await entityPolicy.enforce(readerItem.data);
await DatabaseManager.refreshSingleEntity(
database,
location.id,
@@ -100,15 +99,11 @@ export class DatabaseManager {
entity.metadata!.name,
);
} catch (error) {
let entityName;
if (error instanceof ParserError) {
entityName = error.entityName;
}
await DatabaseManager.logUpdateFailure(
database,
location.id,
error,
entityName,
readerItem.data.metadata?.name,
);
}
}
@@ -129,7 +124,7 @@ export class DatabaseManager {
private static async refreshSingleEntity(
database: Database,
locationId: string,
entity: DescriptorEnvelope,
entity: Entity,
logger: Logger,
): Promise<void> {
const { kind } = entity;
@@ -163,10 +158,7 @@ export class DatabaseManager {
});
}
private static entitiesAreEqual(
first: DescriptorEnvelope,
second: DescriptorEnvelope,
) {
private static entitiesAreEqual(first: Entity, second: Entity) {
const firstClone = lodash.cloneDeep(first);
const secondClone = lodash.cloneDeep(second);
plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts
+4 -1
View File
@@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise<any> {
return knex.schema.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table
.dateTime('created_at')
.defaultTo(knex.fn.now())
.notNullable();
table.string('message');
table
.uuid('location_id')
plugins/catalog-backend/src/database/search.test.ts
+3 -3
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { DescriptorEnvelope } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { buildEntitySearch, visitEntityPart } from './search';
import { DbEntitiesSearchRow } from './types';
@@ -99,7 +99,7 @@ describe('search', () => {
describe('buildEntitySearch', () => {
it('adds special keys even if missing', () => {
const input: DescriptorEnvelope = {
const input: Entity = {
apiVersion: 'a',
kind: 'b',
};
@@ -116,7 +116,7 @@ describe('search', () => {
});
it('adds prefix-stripped versions', () => {
const input: DescriptorEnvelope = {
const input: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
plugins/catalog-backend/src/database/search.ts
+2 -2
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { DescriptorEnvelope } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { DbEntitiesSearchRow } from './types';
// Search entries that start with these prefixes, also get a shorthand without
@@ -119,7 +119,7 @@ export function visitEntityPart(
*/
export function buildEntitySearch(
entityId: string,
entity: DescriptorEnvelope,
entity: Entity,
): DbEntitiesSearchRow[] {
// Start with some special keys that are always present because you want to
// be able to easily search for null specifically
plugins/catalog-backend/src/database/types.ts
+3 -3
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion';
export type DbEntitiesRow = {
id: string;
@@ -32,12 +32,12 @@ export type DbEntitiesRow = {
export type DbEntityRequest = {
locationId?: string;
entity: DescriptorEnvelope;
entity: Entity;
};
export type DbEntityResponse = {
locationId?: string;
entity: DescriptorEnvelope;
entity: Entity;
};
export type DbEntitiesSearchRow = {
plugins/catalog-backend/src/ingestion/DescriptorParsers.ts
-53
View File
@@ -1,53 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { makeValidator } from '../validation';
import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser';
import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser';
import {
DescriptorEnvelope,
DescriptorParser,
KindParser,
ParserError,
} from './types';
export class DescriptorParsers implements DescriptorParser {
static create(): DescriptorParser {
const validators = makeValidator();
return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [
new ComponentDescriptorV1beta1Parser(),
]);
}
constructor(
private readonly envelopeParser: DescriptorEnvelopeParser,
private readonly kindParsers: KindParser[],
) {}
async parse(descriptor: object): Promise<DescriptorEnvelope> {
const envelope = await this.envelopeParser.parse(descriptor);
for (const parser of this.kindParsers) {
const parsed = await parser.tryParse(envelope);
if (parsed) {
return parsed;
}
}
throw new ParserError(
`Unsupported object ${envelope.apiVersion}, ${envelope.kind}`,
envelope.metadata?.name,
);
}
}
plugins/catalog-backend/src/ingestion/IngestionModels.ts
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model';
import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types';
import { LocationReader, LocationReaders } from './source';
import { IngestionModel } from './types';
import { DescriptorParsers } from './descriptor';
export class IngestionModels implements IngestionModel {
private readonly reader: LocationReader;
private readonly parser: DescriptorParser;
private readonly entityPolicy: EntityPolicy;
static default(): IngestionModel {
return new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
}
constructor(
reader: LocationReader,
parser: DescriptorParser,
entityPolicy: EntityPolicy,
) {
this.reader = reader;
this.parser = parser;
this.entityPolicy = entityPolicy;
}
async readLocation(type: string, target: string): Promise<ReaderOutput[]> {
const buffer = await this.reader.tryRead(type, target);
if (!buffer) {
throw new Error(`No reader could handle location ${type} ${target}`);
}
const items = await this.parser.tryParse(buffer);
if (!items) {
throw new Error(`No parser could handle location ${type} ${target}`);
}
const result: ReaderOutput[] = [];
for (const item of items) {
if (item.type === 'error') {
result.push(item);
} else {
try {
const output = await this.entityPolicy.enforce(item.data);
result.push({ type: 'data', data: output });
} catch (e) {
result.push({ type: 'error', error: e });
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/LocationReaders.ts
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationSource } from './sources/FileLocationSource';
import { GitHubLocationSource } from './sources/GitHubLocationSource';
import { LocationReader, LocationSource, ReaderOutput } from './types';
export class LocationReaders implements LocationReader {
static create(): LocationReader {
return new LocationReaders({
file: new FileLocationSource(),
github: new GitHubLocationSource(),
});
}
constructor(private readonly sources: Record<string, LocationSource>) {}
async read(type: string, target: string): Promise<ReaderOutput[]> {
const source = this.sources[type];
if (!source) {
throw new Error(`Unknown location type ${type}`);
}
return source.read(target);
}
}
plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DescriptorParser, ReaderOutput } from './parsers/types';
import { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
/**
* Parses raw descriptor data (e.g. from a file or stream) into entities.
*/
export class DescriptorParsers implements DescriptorParser {
private readonly parsers: DescriptorParser[];
static defaultParsers(): DescriptorParser[] {
return [new YamlDescriptorParser()];
}
constructor(
parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(),
) {
this.parsers = parsers;
}
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
for (const parser of this.parsers) {
const result = await parser.tryParse(data);
if (result) {
return result;
}
}
throw new Error(`Unsupported descriptor format`);
}
}
plugins/catalog-backend/src/ingestion/descriptor/index.ts
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { DescriptorParsers } from './DescriptorParsers';
export { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts
+64
View File
@@ -0,0 +1,64 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import yaml from 'yaml';
import { DescriptorParser, ReaderOutput } from './types';
/**
* Parses descriptors on YAML format
*/
export class YamlDescriptorParser implements DescriptorParser {
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
// TODO(freben): Should perhaps first do format detection, so the parse
// failure can be emitted as a proper error instead of just as if we
// weren't handling the format at all.
let documents;
try {
documents = yaml.parseAllDocuments(data.toString('utf8'));
} catch (e) {
return undefined;
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json as Entity,
});
}
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: Entity };
/**
* Parses raw descriptor data (e.g. from a file) into entities.
*/
export type DescriptorParser = {
/**
* Try to parse some raw data into an entity.
*
* Note that this is only the low level operation of parsing the raw file
* format, e.g. reading JSON or YAML or similar and emitting as structured
* but unvalidated data. The actual validation is performed by EntityPolicy
* and KindParser.
*
* @param data Raw descriptor data
* @returns A list of raw unvalidated entities / errors, or undefined if the
* given data is not meant to be handled by this parser
* @throws An Error if the format was handled and found to not be properly
* formed
*/
tryParse(data: Buffer): Promise<ReaderOutput[] | undefined>;
};
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts
-172
View File
@@ -1,172 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { makeValidator } from '../../validation';
import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser';
describe('DescriptorEnvelopeParser', () => {
let data: any;
let parser: DescriptorEnvelopeParser;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
parser = new DescriptorEnvelopeParser(makeValidator());
});
it('works for the happy path', async () => {
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects missing apiVersion', async () => {
delete data.apiVersion;
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects wrong root type', async () => {
await expect(parser.parse(7)).rejects.toThrow(/object/);
});
it('rejects bad apiVersion', async () => {
data.apiVersion = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects missing kind', async () => {
delete data.kind;
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('rejects bad kind', async () => {
data.kind = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('accepts missing metadata', async () => {
delete data.medatata;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object metadata', async () => {
data.metadata = 7;
await expect(parser.parse(data)).rejects.toThrow(/metadata/);
});
it('accepts missing uid', async () => {
delete data.metadata.uid;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad uid', async () => {
data.metadata.uid = 7;
await expect(parser.parse(data)).rejects.toThrow(/uid/);
});
it('accepts missing etag', async () => {
delete data.metadata.etag;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad etag', async () => {
data.metadata.etag = 7;
await expect(parser.parse(data)).rejects.toThrow(/etag/);
});
it('accepts missing generation', async () => {
delete data.metadata.generation;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad generation', async () => {
data.metadata.generation = 'a';
await expect(parser.parse(data)).rejects.toThrow(/generation/);
});
it('accepts missing spec', async () => {
delete data.spec;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object spec', async () => {
data.spec = 7;
await expect(parser.parse(data)).rejects.toThrow(/spec/);
});
it('rejects bad name', async () => {
data.metadata.name = 7;
await expect(parser.parse(data)).rejects.toThrow(/name/);
});
it('rejects bad namespace', async () => {
data.metadata.namespace = 7;
await expect(parser.parse(data)).rejects.toThrow(/namespace/);
});
it('rejects bad label key', async () => {
data.metadata.labels['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/label.*key/i);
});
it('rejects bad label value', async () => {
data.metadata.labels.a = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/label.*value/i);
});
it('rejects bad annotation key', async () => {
data.metadata.annotations['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i);
});
it('rejects bad annotation value', async () => {
data.metadata.annotations.a = [];
await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i);
});
it('rejects unknown root keys', async () => {
data.spec2 = {};
await expect(parser.parse(data)).rejects.toThrow(/spec2/i);
});
it('rejects reserved keys in the spec root', async () => {
data.spec.apiVersion = 'a/b';
await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i);
});
it('rejects reserved keys in labels', async () => {
data.metadata.labels.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i);
});
it('rejects reserved keys in annotations', async () => {
data.metadata.annotations.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i);
});
});
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts
-206
View File
@@ -1,206 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { Validators } from '../../validation';
import { DescriptorEnvelope } from '../types';
/**
* Parses some raw structured data as a descriptor envelope
*/
export class DescriptorEnvelopeParser {
private schema: yup.Schema<DescriptorEnvelope>;
constructor(validators: Validators) {
const apiVersionSchema = yup
.string()
.required()
.test(
'apiVersion',
'The apiVersion is not formatted according to schema',
validators.isValidApiVersion,
);
const kindSchema = yup
.string()
.required()
.test(
'kind',
'The kind is not formatted according to schema',
validators.isValidKind,
);
const uidSchema = yup
.string()
.notRequired()
.test(
'metadata.uid',
'The uid is not formatted according to schema',
value => value === undefined || value.length > 0,
);
const etagSchema = yup
.string()
.notRequired()
.test(
'metadata.etag',
'The etag value is not according to schema',
value => value === undefined || value.length > 0,
);
const generationSchema = yup
.number()
.notRequired()
.test(
'metadata.generation',
'The generation value is not according to schema',
value => value === undefined || value > 0,
);
const nameSchema = yup
.string()
.notRequired()
.test(
'metadata.name',
'The name is not formatted according to schema',
value => value === undefined || validators.isValidEntityName(value),
);
const namespaceSchema = yup
.string()
.notRequired()
.test(
'metadata.namespace',
'The namespace is malformed',
value => value === undefined || validators.isValidNamespace(value),
);
const labelsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.labels.keys',
message: 'Label keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidLabelKey)
);
},
})
.test({
name: 'metadata.labels.values',
message: 'Label values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidLabelValue)
);
},
});
const annotationsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.annotations.keys',
message: 'Annotation keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidAnnotationKey)
);
},
})
.test({
name: 'metadata.annotations.values',
message: 'Annotation values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidAnnotationValue)
);
},
});
const metadataSchema = yup
.object({
uid: uidSchema,
etag: etagSchema,
generation: generationSchema,
name: nameSchema,
namespace: namespaceSchema,
labels: labelsSchema,
annotations: annotationsSchema,
})
.notRequired();
const specSchema = yup.object({}).notRequired();
this.schema = yup
.object({
apiVersion: apiVersionSchema,
kind: kindSchema,
metadata: metadataSchema,
spec: specSchema,
})
.noUnknown();
}
async parse(data: any): Promise<DescriptorEnvelope> {
let result: DescriptorEnvelope;
try {
result = await this.schema.validate(data, { strict: true });
} catch (e) {
throw new Error(`Malformed envelope, ${e}`);
}
// These are keys with specific semantic meaning in a document, that we do
// not want to appear in the root of the spec, or as labels or as
// annotations, because they will lead to confusion.
const reservedKeys = [
'apiVersion',
'kind',
'uid',
'etag',
'generation',
'name',
'namespace',
'labels',
'annotations',
'spec',
];
for (const key of reservedKeys) {
if (result.spec?.hasOwnProperty(key)) {
throw new Error(
`The spec may not contain the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.labels?.hasOwnProperty(key)) {
throw new Error(
`A label may not have the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.annotations?.hasOwnProperty(key)) {
throw new Error(
`An annotation may not have the key ${key}, because it has reserved meaning`,
);
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/index.ts
+4 -3
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export * from './DescriptorParsers';
export * from './LocationReaders';
export * from './types';
export * from './descriptor';
export { IngestionModels } from './IngestionModels';
export * from './source';
export type { IngestionModel } from './types';
plugins/catalog-backend/src/ingestion/source/LocationReaders.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationReader } from './readers/FileLocationReader';
import { GitHubLocationReader } from './readers/GitHubLocationReader';
import { LocationReader } from './readers/types';
export class LocationReaders implements LocationReader {
private readonly readers: LocationReader[];
static defaultReaders(): LocationReader[] {
return [new FileLocationReader(), new GitHubLocationReader()];
}
constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) {
this.readers = readers;
}
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
for (const reader of this.readers) {
const result = await reader.tryRead(type, target);
if (result) {
return result;
}
}
throw new Error(`Could not read unknown location "${type}", "${target}"`);
}
}
plugins/catalog-backend/src/ingestion/source/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { LocationReaders } from './LocationReaders';
export { FileLocationReader } from './readers/FileLocationReader';
export { GitHubLocationReader } from './readers/GitHubLocationReader';
export type { LocationReader } from './readers/types';
plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts → plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts
+11 -12
View File
@@ -15,22 +15,21 @@
*/
import fs from 'fs-extra';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
import { LocationReader } from './types';
/**
* Reads a file from the local file system.
*/
export class FileLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'file') {
return undefined;
}
export class FileLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let rawYaml;
try {
rawYaml = await fs.readFile(target, 'utf8');
return await fs.readFile(target);
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts → plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts
+17 -35
View File
@@ -16,43 +16,26 @@
jest.mock('node-fetch');
import fs from 'fs-extra';
import fetch from 'node-fetch';
import path from 'path';
import { GitHubLocationSource } from '../GitHubLocationSource';
import { GitHubLocationReader } from './GitHubLocationReader';
const { Response } = jest.requireActual('node-fetch');
const FIXTURES_DIR = path.resolve(
__dirname,
'..',
'..',
'..',
'..',
'fixtures',
);
const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => {
acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8');
return acc;
}, {} as Record<string, string>);
describe('Unit: GitHubLocationSource', () => {
describe('Unit: GitHubLocationReader', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('fetches the file and parses it correctly', async () => {
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures['one_component.yaml'])),
);
const reader = new GitHubLocationSource();
(fetch as any).mockResolvedValueOnce(new Response('hello'));
const result = await reader.read(
const reader = new GitHubLocationReader();
const buffer = await reader.tryRead(
'github',
'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
expect(buffer?.toString('utf8')).toBe('hello');
});
it('changes the url to point to https://raw.githubusercontent.com', async () => {
@@ -61,12 +44,12 @@ describe('Unit: GitHubLocationSource', () => {
const folderPath = `master/plugins/catalog-backend/fixtures`;
const componentFilename = `one_component.yaml`;
const rawGitHubUrl = `https://raw.githubusercontent.com`;
const reader = new GitHubLocationSource();
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures[componentFilename])),
);
await reader.read(
const reader = new GitHubLocationReader();
(fetch as any).mockResolvedValueOnce(new Response('hello'));
await reader.tryRead(
'github',
`${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
);
@@ -76,7 +59,7 @@ describe('Unit: GitHubLocationSource', () => {
});
describe('rejects wrong urls', () => {
const reader = new GitHubLocationSource();
const reader = new GitHubLocationReader();
it.each([
['http://example.com/one_component.yaml'],
@@ -87,7 +70,7 @@ describe('Unit: GitHubLocationSource', () => {
])(
'%p',
async (url: string) =>
await expect(reader.read(url)).rejects.toThrow(/url/),
await expect(reader.tryRead('github', url)).rejects.toThrow(/url/),
);
});
});
@@ -100,11 +83,10 @@ describe('Integration: GitHubLocationSource', () => {
it('fetches the fixture from backstage repo', async () => {
const PERMANENT_LINK =
'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
const reader = new GitHubLocationSource();
const result = await reader.read(PERMANENT_LINK);
const reader = new GitHubLocationReader();
const result = await reader.tryRead('github', PERMANENT_LINK);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
expect(result?.toString('utf8')).toContain('component3');
});
});
plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts → plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts
+24 -23
View File
@@ -16,17 +16,31 @@
import fetch from 'node-fetch';
import { URL } from 'url';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
import { LocationReader } from './types';
// Pointing to raw.githubusercontent.com for now
// to be changed in the future, after auth and tokens are done
export class GitHubLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let url: URL;
/**
* Reads a file whose target is a GitHub URL.
*
* Uses raw.githubusercontent.com for now, but this will probably change in the
* future when token auth is implemented.
*/
export class GitHubLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'github') {
return undefined;
}
const url = this.buildRawUrl(target);
try {
url = new URL(target);
return await fetch(url.toString()).then(x => x.buffer());
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
}
private buildRawUrl(target: string): URL {
try {
const url = new URL(target);
const [
empty,
@@ -51,23 +65,10 @@ export class GitHubLocationSource implements LocationSource {
url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
url.hostname = 'raw.githubusercontent.com';
url.protocol = 'https';
return url;
} catch (e) {
throw new Error(`Incorrect url: ${target}, ${e}`);
}
let rawYaml;
try {
rawYaml = await fetch(url.toString()).then(x => {
return x.text();
});
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/source/readers/types.ts
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The target contents, as a raw Buffer, or undefined if this type
* was not meant to be consumed by this reader
* @throws An error if the type was meant for this reader, but could not be
* read
*/
tryRead(type: string, target: string): Promise<Buffer | undefined>;
};
plugins/catalog-backend/src/ingestion/sources/util.ts
-55
View File
@@ -1,55 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { ReaderOutput } from '../types';
export function readDescriptorYaml(data: string): ReaderOutput[] {
let documents;
try {
documents = yaml.parseAllDocuments(data);
} catch (e) {
throw new Error(`Could not parse YAML data, ${e}`);
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json,
});
}
}
}
}
return result;
}
plugins/catalog-backend/src/ingestion/types.ts
+3 -167
View File
@@ -14,172 +14,8 @@
* limitations under the License.
*/
import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser';
import { ReaderOutput } from './descriptor/parsers/types';
export type ComponentDescriptor = ComponentDescriptorV1beta1;
/**
* Metadata fields common to all versions/kinds of entity.
*
* @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
*/
export type EntityMeta = {
/**
* A globally unique ID for the entity.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, but the server is free to reject requests
* that do so in such a way that it breaks semantics.
*/
uid?: string;
/**
* An opaque string that changes for each update operation to any part of
* the entity, including metadata.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, and the server will then reject the
* operation if it does not match the current stored value.
*/
etag?: string;
/**
* A positive nonzero number that indicates the current generation of data
* for this entity; the value is incremented each time the spec changes.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations.
*/
generation?: number;
/**
* The name of the entity.
*
* Must be uniqe within the catalog at any given point in time, for any
* given namespace, for any given kind.
*/
name?: string;
/**
* The namespace that the entity belongs to.
*/
namespace?: string;
/**
* Key/value pairs of identifying information attached to the entity.
*/
labels?: Record<string, string>;
/**
* Key/value pairs of non-identifying auxiliary information attached to the
* entity.
*/
annotations?: Record<string, string>;
};
/**
* The format envelope that's common to all versions/kinds.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
*/
export type DescriptorEnvelope = {
/**
* The version of specification format for this particular entity that
* this is written against.
*/
apiVersion: string;
/**
* The high level entity type being described.
*/
kind: string;
/**
* Optional metadata related to the entity.
*/
metadata?: EntityMeta;
/**
* The specification data describing the entity itself.
*/
spec?: object;
};
/**
* Parses and validates descriptors.
*
* The output must be validated and well formed.
*/
export type DescriptorParser = {
/**
* Parses and validates a single raw descriptor.
*
* @param descriptor A raw descriptor object
* @returns A structure describing the parsed and validated descriptor
* @throws An Error if the descriptor was malformed
*/
parse(descriptor: object): Promise<DescriptorEnvelope>;
};
/**
* Parses and validates a single envelope into its materialized kind.
*
* These parsers may assume that the envelope is already validated and well
* formed.
*/
export type KindParser = {
/**
* Try to parse an envelope into a materialized kind.
*
* @param envelope A valid descriptor envelope
* @returns A materialized type, or undefined if the given version/kind is
* not meant to be handled by this parser
* @throws An Error if the type was handled and found to not be properly
* formatted
*/
tryParse(
envelope: DescriptorEnvelope,
): Promise<DescriptorEnvelope | undefined>;
};
export class ParserError extends Error {
constructor(message?: string, private _entityName?: string | undefined) {
super(message);
}
get entityName() {
return this._entityName;
}
}
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: object };
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The parsed contents, as an array of unverified descriptors or
* errors where the individual documents could not be parsed.
* @throws An error if the location as a whole could not be read
*/
read(type: string, target: string): Promise<ReaderOutput[]>;
};
export type LocationSource = {
/**
* Reads the contents of a single location.
*
* @param target The location target to read
* @returns The parsed contents, as an array of unverified descriptors
* @throws An error if the location target could not be read
*/
read(target: string): Promise<ReaderOutput[]>;
export type IngestionModel = {
readLocation(type: string, target: string): Promise<ReaderOutput[]>;
};
plugins/catalog-backend/src/service/router.test.ts
+7 -5
View File
@@ -15,10 +15,10 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import request from 'supertest';
import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog';
import { DescriptorEnvelope } from '../ingestion';
import { createRouter } from './router';
class MockEntitiesCatalog implements EntitiesCatalog {
@@ -37,7 +37,7 @@ class MockLocationsCatalog implements LocationsCatalog {
describe('createRouter', () => {
describe('entities', () => {
it('happy path: lists entities', async () => {
const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }];
const entities: Entity[] = [{ apiVersion: 'a', kind: 'b' }];
const catalog = new MockEntitiesCatalog();
catalog.entities.mockResolvedValueOnce(entities);
@@ -76,7 +76,7 @@ describe('createRouter', () => {
describe('entityByUid', () => {
it('can fetch entity by uid', async () => {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
@@ -117,7 +117,7 @@ describe('createRouter', () => {
describe('entityByName', () => {
it('can fetch entity by name', async () => {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
@@ -190,7 +190,9 @@ describe('createRouter', () => {
});
const app = express().use(router);
const response = await request(app).post('/locations').send(location);
const response = await request(app)
.post('/locations')
.send(location);
expect(response.status).toEqual(400);
});
plugins/catalog-backend/src/service/router.ts
+1 -1
View File
@@ -22,7 +22,7 @@ import {
addLocationSchema,
EntitiesCatalog,
EntityFilters,
LocationsCatalog,
LocationsCatalog
} from '../catalog';
import { validateRequestBody } from './util';