paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 54 Packages Projects Releases Wiki Activity

Move the model into a plugin-catalog-model for sharing outside the backend

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-05-28 11:52:50 +02:00
parent add55f64a8
commit 1d6324a92d
35 changed files with 119 additions and 1730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/package.json
+9 -4
View File
@@ -10,7 +10,7 @@
},
"scripts": {
"build": "tsc",
"start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess nodemon",
"start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"nodemon -r esm\\\"",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"clean": "backstage-cli clean",
@@ -18,13 +18,15 @@
},
"dependencies": {
"@backstage/backend-common": "^0.1.1-alpha.6",
"@backstage/catalog-model": "^0.1.1-alpha.6",
"@backstage/plugin-auth-backend": "^0.1.1-alpha.6",
"@backstage/plugin-catalog-backend": "^0.1.1-alpha.6",
"@backstage/plugin-sentry-backend": "^0.1.1-alpha.6",
"@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6",
"@backstage/plugin-identity-backend": "^0.1.1-alpha.6",
"@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.6",
"@backstage/plugin-sentry-backend": "^0.1.1-alpha.6",
"compression": "^1.7.4",
"cors": "^2.8.5",
"esm": "^3.2.25",
"express": "^4.17.1",
"helmet": "^3.22.0",
"knex": "^0.21.1",
@@ -43,6 +45,9 @@
"typescript": "^3.9.2"
},
"nodemonConfig": {
"watch": "./dist"
"watch": [
"./dist",
"node_modules/@backstage*"
]
}
}
packages/backend/src/plugins/catalog.ts
+10 -4
View File
@@ -21,22 +21,28 @@ import {
DatabaseManager,
DescriptorParsers,
LocationReaders,
IngestionModels,
runPeriodically,
} from '@backstage/plugin-catalog-backend';
import { PluginEnvironment } from '../types';
import { EntityPolicies } from '@backstage/catalog-model';
export default async function ({ logger, database }: PluginEnvironment) {
const reader = LocationReaders.create();
const parser = DescriptorParsers.create();
const policy = new EntityPolicies();
const ingestion = new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
const db = await DatabaseManager.createDatabase(database, logger);
runPeriodically(
() => DatabaseManager.refreshLocations(db, reader, parser, logger),
() => DatabaseManager.refreshLocations(db, ingestion, policy, logger),
10000,
);
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db, reader);
const locationsCatalog = new DatabaseLocationsCatalog(db, ingestion);
return await createRouter({ entitiesCatalog, locationsCatalog, logger });
}
plugins/catalog-backend/package.json
+3 -2
View File
@@ -16,8 +16,7 @@
},
"dependencies": {
"@backstage/backend-common": "^0.1.1-alpha.6",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@backstage/catalog-model": "^0.1.1-alpha.6",
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
@@ -38,6 +37,8 @@
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.6",
"@types/lodash": "^4.14.151",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
"@types/yup": "^0.28.2",
"jest-fetch-mock": "^3.0.3",
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+1 -1
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { Database } from '../database';
import { Entity } from '../ingestion/types';
import { EntitiesCatalog, EntityFilters } from './types';
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts
+19 -16
View File
@@ -14,12 +14,27 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import knex from 'knex';
import path from 'path';
import { Database } from '../database';
import { ReaderOutput } from '../ingestion/types';
import { IngestionModel } from '../ingestion/types';
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
class MockIngestionModel implements IngestionModel {
readLocation = jest.fn(async (type: string, target: string) => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return [{ type: 'data', data: {} as Entity } as const];
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
});
}
describe('DatabaseLocationsCatalog', () => {
const database = knex({
client: 'sqlite3',
@@ -31,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => {
});
let db: Database;
let catalog: DatabaseLocationsCatalog;
const mockLocationReader = {
read: async (type: string, target: string): Promise<ReaderOutput[]> => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return Promise.resolve([{ type: 'data', data: {} }]);
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
},
};
let ingestionModel: IngestionModel;
beforeEach(async () => {
await database.migrate.latest({
@@ -52,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => {
loadExtensions: ['.ts'],
});
db = new Database(database, getVoidLogger());
catalog = new DatabaseLocationsCatalog(db, mockLocationReader);
ingestionModel = new MockIngestionModel();
catalog = new DatabaseLocationsCatalog(db, ingestionModel);
});
it('resolves to location with id', async () => {
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts
+11 -3
View File
@@ -15,17 +15,25 @@
*/
import { Database } from '../database';
import { LocationReader } from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { AddLocation, Location, LocationsCatalog } from './types';
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(
private readonly database: Database,
private readonly reader: LocationReader,
private readonly ingestionModel: IngestionModel,
) {}
async addLocation(location: AddLocation): Promise<Location> {
const outputs = await this.reader.read(location.type, location.target);
const outputs = await this.ingestionModel.readLocation(
location.type,
location.target,
);
if (!outputs) {
throw new Error(
`Unknown location type ${location.type} ${location.target}`,
);
}
outputs.forEach(output => {
if (output.type === 'error') {
throw new Error(
plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
+1 -1
View File
@@ -15,8 +15,8 @@
*/
import { NotFoundError } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import lodash from 'lodash';
import { Entity } from '../ingestion';
import { EntitiesCatalog } from './types';
export class StaticEntitiesCatalog implements EntitiesCatalog {
plugins/catalog-backend/src/catalog/types.ts
+1 -1
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { Entity } from '../ingestion';
//
// Entities
plugins/catalog-backend/src/database/Database.test.ts
+1 -1
View File
@@ -19,9 +19,9 @@ import {
getVoidLogger,
NotFoundError,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import Knex from 'knex';
import path from 'path';
import { Entity } from '../ingestion';
import { Database } from './Database';
import {
AddDatabaseLocation,
plugins/catalog-backend/src/database/Database.ts
+1 -1
View File
@@ -19,12 +19,12 @@ import {
InputError,
NotFoundError,
} from '@backstage/backend-common';
import { Entity, EntityMeta } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntityFilters } from '../catalog';
import { Entity, EntityMeta } from '../ingestion';
import { buildEntitySearch } from './search';
import {
AddDatabaseLocation,
plugins/catalog-backend/src/database/DatabaseManager.test.ts
+40 -42
View File
@@ -15,13 +15,9 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import {
ComponentDescriptor,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
@@ -32,18 +28,18 @@ describe('DatabaseManager', () => {
const db = ({
locations: jest.fn().mockResolvedValue([]),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(),
const reader: IngestionModel = {
readLocation: jest.fn(),
};
const parser: DescriptorParser = {
parse: jest.fn(),
const policy: EntityPolicy = {
apply: jest.fn(),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).not.toHaveBeenCalled();
expect(parser.parse).not.toHaveBeenCalled();
expect(reader.readLocation).not.toHaveBeenCalled();
expect(policy.apply).not.toHaveBeenCalled();
});
it('can update a single location', async () => {
@@ -52,7 +48,7 @@ describe('DatabaseManager', () => {
type: 'some',
target: 'thing',
};
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
@@ -68,18 +64,20 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as Partial<Database>) as Database;
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
apply: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).toHaveBeenCalledTimes(1);
expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(reader.readLocation).toHaveBeenCalledTimes(1);
expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(db.addEntity).toHaveBeenCalledTimes(1);
expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, {
locationId: '123',
@@ -108,21 +106,23 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
apply: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -158,23 +158,23 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const policy: EntityPolicy = {
apply: jest.fn(() => Promise.reject(new Error('parser error message'))),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -211,19 +211,17 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(() =>
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.reject([{ type: 'error', error: new Error('test message') }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
),
const policy: EntityPolicy = {
apply: jest.fn(() => Promise.reject(new Error('parser error message'))),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
plugins/catalog-backend/src/database/DatabaseManager.ts
+10 -15
View File
@@ -14,16 +14,12 @@
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import path from 'path';
import { Logger } from 'winston';
import {
DescriptorParser,
Entity,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types';
@@ -67,8 +63,8 @@ export class DatabaseManager {
public static async refreshLocations(
database: Database,
reader: LocationReader,
parser: DescriptorParser,
ingestionModel: IngestionModel,
entityPolicy: EntityPolicy,
logger: Logger,
): Promise<void> {
const locations = await database.locations();
@@ -78,7 +74,10 @@ export class DatabaseManager {
`Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
);
const readerOutput = await reader.read(location.type, location.target);
const readerOutput = await ingestionModel.readLocation(
location.type,
location.target,
);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
@@ -87,7 +86,7 @@ export class DatabaseManager {
}
try {
const entity = await parser.parse(readerItem.data);
const entity = await entityPolicy.apply(readerItem.data);
await DatabaseManager.refreshSingleEntity(
database,
location.id,
@@ -100,15 +99,11 @@ export class DatabaseManager {
entity.metadata!.name,
);
} catch (error) {
let entityName;
if (error instanceof ParserError) {
entityName = error.entityName;
}
await DatabaseManager.logUpdateFailure(
database,
location.id,
error,
entityName,
readerItem.data.metadata?.name,
);
}
}
plugins/catalog-backend/src/database/search.test.ts
+1 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Entity } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { buildEntitySearch, visitEntityPart } from './search';
import { DbEntitiesSearchRow } from './types';
plugins/catalog-backend/src/database/search.ts
+1 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Entity } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { DbEntitiesSearchRow } from './types';
// Search entries that start with these prefixes, also get a shorthand without
plugins/catalog-backend/src/database/types.ts
+1 -1
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { Entity } from '../ingestion';
export type DbEntitiesRow = {
id: string;
plugins/catalog-backend/src/ingestion/DescriptorParsers.ts
-48
View File
@@ -1,48 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { makeValidator } from '../validation';
import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser';
import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser';
import { DescriptorParser, Entity, KindParser, ParserError } from './types';
export class DescriptorParsers implements DescriptorParser {
static create(): DescriptorParser {
const validators = makeValidator();
return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [
new ComponentDescriptorV1beta1Parser(),
]);
}
constructor(
private readonly envelopeParser: DescriptorEnvelopeParser,
private readonly kindParsers: KindParser[],
) {}
async parse(descriptor: object): Promise<Entity> {
const envelope = await this.envelopeParser.parse(descriptor);
for (const parser of this.kindParsers) {
const parsed = await parser.tryParse(envelope);
if (parsed) {
return parsed;
}
}
throw new ParserError(
`Unsupported object ${envelope.apiVersion}, ${envelope.kind}`,
envelope.metadata?.name,
);
}
}
plugins/catalog-backend/src/ingestion/LocationReaders.ts
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationSource } from './sources/FileLocationSource';
import { GitHubLocationSource } from './sources/GitHubLocationSource';
import { LocationReader, LocationSource, ReaderOutput } from './types';
export class LocationReaders implements LocationReader {
static create(): LocationReader {
return new LocationReaders({
file: new FileLocationSource(),
github: new GitHubLocationSource(),
});
}
constructor(private readonly sources: Record<string, LocationSource>) {}
async read(type: string, target: string): Promise<ReaderOutput[]> {
const source = this.sources[type];
if (!source) {
throw new Error(`Unknown location type ${type}`);
}
return source.read(target);
}
}
plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts
-61
View File
@@ -1,61 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { Entity, KindParser, ParserError } from '../types';
export interface ComponentDescriptorV1beta1 extends Entity {
spec: {
type: string;
};
}
export class ComponentDescriptorV1beta1Parser implements KindParser {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<ComponentDescriptorV1beta1>>({
metadata: yup
.object({
name: yup.string().required(),
})
.required(),
spec: yup
.object({
type: yup.string().required(),
})
.required(),
});
}
async tryParse(envelope: Entity): Promise<Entity | undefined> {
if (
envelope.apiVersion !== 'backstage.io/v1beta1' ||
envelope.kind !== 'Component'
) {
return undefined;
}
try {
return await this.schema.validate(envelope, { strict: true });
} catch (e) {
throw new ParserError(
`Malformed component, ${e}`,
envelope.metadata?.name,
);
}
}
}
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts
-172
View File
@@ -1,172 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { makeValidator } from '../../validation';
import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser';
describe('DescriptorEnvelopeParser', () => {
let data: any;
let parser: DescriptorEnvelopeParser;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
parser = new DescriptorEnvelopeParser(makeValidator());
});
it('works for the happy path', async () => {
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects missing apiVersion', async () => {
delete data.apiVersion;
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects wrong root type', async () => {
await expect(parser.parse(7)).rejects.toThrow(/object/);
});
it('rejects bad apiVersion', async () => {
data.apiVersion = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects missing kind', async () => {
delete data.kind;
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('rejects bad kind', async () => {
data.kind = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('accepts missing metadata', async () => {
delete data.medatata;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object metadata', async () => {
data.metadata = 7;
await expect(parser.parse(data)).rejects.toThrow(/metadata/);
});
it('accepts missing uid', async () => {
delete data.metadata.uid;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad uid', async () => {
data.metadata.uid = 7;
await expect(parser.parse(data)).rejects.toThrow(/uid/);
});
it('accepts missing etag', async () => {
delete data.metadata.etag;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad etag', async () => {
data.metadata.etag = 7;
await expect(parser.parse(data)).rejects.toThrow(/etag/);
});
it('accepts missing generation', async () => {
delete data.metadata.generation;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad generation', async () => {
data.metadata.generation = 'a';
await expect(parser.parse(data)).rejects.toThrow(/generation/);
});
it('accepts missing spec', async () => {
delete data.spec;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object spec', async () => {
data.spec = 7;
await expect(parser.parse(data)).rejects.toThrow(/spec/);
});
it('rejects bad name', async () => {
data.metadata.name = 7;
await expect(parser.parse(data)).rejects.toThrow(/name/);
});
it('rejects bad namespace', async () => {
data.metadata.namespace = 7;
await expect(parser.parse(data)).rejects.toThrow(/namespace/);
});
it('rejects bad label key', async () => {
data.metadata.labels['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/label.*key/i);
});
it('rejects bad label value', async () => {
data.metadata.labels.a = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/label.*value/i);
});
it('rejects bad annotation key', async () => {
data.metadata.annotations['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i);
});
it('rejects bad annotation value', async () => {
data.metadata.annotations.a = [];
await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i);
});
it('rejects unknown root keys', async () => {
data.spec2 = {};
await expect(parser.parse(data)).rejects.toThrow(/spec2/i);
});
it('rejects reserved keys in the spec root', async () => {
data.spec.apiVersion = 'a/b';
await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i);
});
it('rejects reserved keys in labels', async () => {
data.metadata.labels.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i);
});
it('rejects reserved keys in annotations', async () => {
data.metadata.annotations.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i);
});
});
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts
-206
View File
@@ -1,206 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { Validators } from '../../validation';
import { Entity } from '../types';
/**
* Parses some raw structured data as a descriptor envelope
*/
export class DescriptorEnvelopeParser {
private schema: yup.Schema<Entity>;
constructor(validators: Validators) {
const apiVersionSchema = yup
.string()
.required()
.test(
'apiVersion',
'The apiVersion is not formatted according to schema',
validators.isValidApiVersion,
);
const kindSchema = yup
.string()
.required()
.test(
'kind',
'The kind is not formatted according to schema',
validators.isValidKind,
);
const uidSchema = yup
.string()
.notRequired()
.test(
'metadata.uid',
'The uid is not formatted according to schema',
value => value === undefined || value.length > 0,
);
const etagSchema = yup
.string()
.notRequired()
.test(
'metadata.etag',
'The etag value is not according to schema',
value => value === undefined || value.length > 0,
);
const generationSchema = yup
.number()
.notRequired()
.test(
'metadata.generation',
'The generation value is not according to schema',
value => value === undefined || value > 0,
);
const nameSchema = yup
.string()
.notRequired()
.test(
'metadata.name',
'The name is not formatted according to schema',
value => value === undefined || validators.isValidEntityName(value),
);
const namespaceSchema = yup
.string()
.notRequired()
.test(
'metadata.namespace',
'The namespace is malformed',
value => value === undefined || validators.isValidNamespace(value),
);
const labelsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.labels.keys',
message: 'Label keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidLabelKey)
);
},
})
.test({
name: 'metadata.labels.values',
message: 'Label values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidLabelValue)
);
},
});
const annotationsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.annotations.keys',
message: 'Annotation keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidAnnotationKey)
);
},
})
.test({
name: 'metadata.annotations.values',
message: 'Annotation values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidAnnotationValue)
);
},
});
const metadataSchema = yup
.object({
uid: uidSchema,
etag: etagSchema,
generation: generationSchema,
name: nameSchema,
namespace: namespaceSchema,
labels: labelsSchema,
annotations: annotationsSchema,
})
.notRequired();
const specSchema = yup.object({}).notRequired();
this.schema = yup
.object({
apiVersion: apiVersionSchema,
kind: kindSchema,
metadata: metadataSchema,
spec: specSchema,
})
.noUnknown();
}
async parse(data: any): Promise<Entity> {
let result: Entity;
try {
result = await this.schema.validate(data, { strict: true });
} catch (e) {
throw new Error(`Malformed envelope, ${e}`);
}
// These are keys with specific semantic meaning in a document, that we do
// not want to appear in the root of the spec, or as labels or as
// annotations, because they will lead to confusion.
const reservedKeys = [
'apiVersion',
'kind',
'uid',
'etag',
'generation',
'name',
'namespace',
'labels',
'annotations',
'spec',
];
for (const key of reservedKeys) {
if (result.spec?.hasOwnProperty(key)) {
throw new Error(
`The spec may not contain the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.labels?.hasOwnProperty(key)) {
throw new Error(
`A label may not have the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.annotations?.hasOwnProperty(key)) {
throw new Error(
`An annotation may not have the key ${key}, because it has reserved meaning`,
);
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/index.ts
+4 -3
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export * from './DescriptorParsers';
export * from './LocationReaders';
export * from './types';
export * from './descriptor';
export { IngestionModels } from './IngestionModels';
export * from './source';
export type { IngestionModel } from './types';
plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts
-36
View File
@@ -1,36 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
export class FileLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let rawYaml;
try {
rawYaml = await fs.readFile(target, 'utf8');
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts
-73
View File
@@ -1,73 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fetch from 'node-fetch';
import { URL } from 'url';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
// Pointing to raw.githubusercontent.com for now
// to be changed in the future, after auth and tokens are done
export class GitHubLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let url: URL;
try {
url = new URL(target);
const [
empty,
userOrOrg,
repoName,
blobKeyword,
...restOfPath
] = url.pathname.split('/');
if (
url.hostname !== 'github.com' ||
empty !== '' ||
userOrOrg === '' ||
repoName === '' ||
blobKeyword !== 'blob' ||
!restOfPath.join('/').match(/\.yaml$/)
) {
throw new Error('Wrong GitHub URL');
}
// Removing the "blob" part
url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
url.hostname = 'raw.githubusercontent.com';
url.protocol = 'https';
} catch (e) {
throw new Error(`Incorrect url: ${target}, ${e}`);
}
let rawYaml;
try {
rawYaml = await fetch(url.toString()).then(x => {
return x.text();
});
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts
-110
View File
@@ -1,110 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('node-fetch');
import fs from 'fs-extra';
import fetch from 'node-fetch';
import path from 'path';
import { GitHubLocationSource } from '../GitHubLocationSource';
const { Response } = jest.requireActual('node-fetch');
const FIXTURES_DIR = path.resolve(
__dirname,
'..',
'..',
'..',
'..',
'fixtures',
);
const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => {
acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8');
return acc;
}, {} as Record<string, string>);
describe('Unit: GitHubLocationSource', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('fetches the file and parses it correctly', async () => {
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures['one_component.yaml'])),
);
const reader = new GitHubLocationSource();
const result = await reader.read(
'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
});
it('changes the url to point to https://raw.githubusercontent.com', async () => {
const gitHubUrl = `https://github.com`;
const project = `spotify/backstage`;
const folderPath = `master/plugins/catalog-backend/fixtures`;
const componentFilename = `one_component.yaml`;
const rawGitHubUrl = `https://raw.githubusercontent.com`;
const reader = new GitHubLocationSource();
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures[componentFilename])),
);
await reader.read(
`${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
);
expect(fetch).toHaveBeenCalledWith(
`${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`,
);
});
describe('rejects wrong urls', () => {
const reader = new GitHubLocationSource();
it.each([
['http://example.com/one_component.yaml'],
['http://github.com/one_component.yaml'],
['http://github.com/PROJECT/one_component.yaml'],
['http://github.com/PROJECT/REPO/one_component.yaml'],
['http://github.com/PROJECT/REPO/one_component.json'],
])(
'%p',
async (url: string) =>
await expect(reader.read(url)).rejects.toThrow(/url/),
);
});
});
describe('Integration: GitHubLocationSource', () => {
beforeAll(() => {
(fetch as any).mockImplementation(jest.requireActual('node-fetch'));
});
it('fetches the fixture from backstage repo', async () => {
const PERMANENT_LINK =
'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
const reader = new GitHubLocationSource();
const result = await reader.read(PERMANENT_LINK);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
});
});
plugins/catalog-backend/src/ingestion/sources/util.ts
-55
View File
@@ -1,55 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { ReaderOutput } from '../types';
export function readDescriptorYaml(data: string): ReaderOutput[] {
let documents;
try {
documents = yaml.parseAllDocuments(data);
} catch (e) {
throw new Error(`Could not parse YAML data, ${e}`);
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json,
});
}
}
}
}
return result;
}
plugins/catalog-backend/src/ingestion/types.ts
+3 -165
View File
@@ -14,170 +14,8 @@
* limitations under the License.
*/
import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser';
import { ReaderOutput } from './descriptor/parsers/types';
export type ComponentDescriptor = ComponentDescriptorV1beta1;
/**
* Metadata fields common to all versions/kinds of entity.
*
* @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
*/
export type EntityMeta = {
/**
* A globally unique ID for the entity.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, but the server is free to reject requests
* that do so in such a way that it breaks semantics.
*/
uid?: string;
/**
* An opaque string that changes for each update operation to any part of
* the entity, including metadata.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, and the server will then reject the
* operation if it does not match the current stored value.
*/
etag?: string;
/**
* A positive nonzero number that indicates the current generation of data
* for this entity; the value is incremented each time the spec changes.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations.
*/
generation?: number;
/**
* The name of the entity.
*
* Must be uniqe within the catalog at any given point in time, for any
* given namespace, for any given kind.
*/
name?: string;
/**
* The namespace that the entity belongs to.
*/
namespace?: string;
/**
* Key/value pairs of identifying information attached to the entity.
*/
labels?: Record<string, string>;
/**
* Key/value pairs of non-identifying auxiliary information attached to the
* entity.
*/
annotations?: Record<string, string>;
};
/**
* The format envelope that's common to all versions/kinds.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
*/
export type Entity = {
/**
* The version of specification format for this particular entity that
* this is written against.
*/
apiVersion: string;
/**
* The high level entity type being described.
*/
kind: string;
/**
* Optional metadata related to the entity.
*/
metadata?: EntityMeta;
/**
* The specification data describing the entity itself.
*/
spec?: object;
};
/**
* Parses and validates descriptors.
*
* The output must be validated and well formed.
*/
export type DescriptorParser = {
/**
* Parses and validates a single raw descriptor.
*
* @param descriptor A raw descriptor object
* @returns A structure describing the parsed and validated descriptor
* @throws An Error if the descriptor was malformed
*/
parse(descriptor: object): Promise<Entity>;
};
/**
* Parses and validates a single envelope into its materialized kind.
*
* These parsers may assume that the envelope is already validated and well
* formed.
*/
export type KindParser = {
/**
* Try to parse an envelope into a materialized kind.
*
* @param envelope A valid descriptor envelope
* @returns A materialized type, or undefined if the given version/kind is
* not meant to be handled by this parser
* @throws An Error if the type was handled and found to not be properly
* formatted
*/
tryParse(envelope: Entity): Promise<Entity | undefined>;
};
export class ParserError extends Error {
constructor(message?: string, private _entityName?: string | undefined) {
super(message);
}
get entityName() {
return this._entityName;
}
}
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: object };
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The parsed contents, as an array of unverified descriptors or
* errors where the individual documents could not be parsed.
* @throws An error if the location as a whole could not be read
*/
read(type: string, target: string): Promise<ReaderOutput[]>;
};
export type LocationSource = {
/**
* Reads the contents of a single location.
*
* @param target The location target to read
* @returns The parsed contents, as an array of unverified descriptors
* @throws An error if the location target could not be read
*/
read(target: string): Promise<ReaderOutput[]>;
export type IngestionModel = {
readLocation(type: string, target: string): Promise<ReaderOutput[]>;
};
plugins/catalog-backend/src/service/router.test.ts
+1 -1
View File
@@ -15,10 +15,10 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import request from 'supertest';
import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog';
import { Entity } from '../ingestion';
import { createRouter } from './router';
class MockEntitiesCatalog implements EntitiesCatalog {
plugins/catalog-backend/src/service/router.ts
+1 -1
View File
@@ -22,7 +22,7 @@ import {
addLocationSchema,
EntitiesCatalog,
EntityFilters,
LocationsCatalog,
LocationsCatalog
} from '../catalog';
import { validateRequestBody } from './util';
plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts
-178
View File
@@ -1,178 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
describe('CommonValidatorFunctions', () => {
describe('isValidPrefixAndOrSuffix', () => {
it('only accepts strings', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
null,
'/',
() => true,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
7,
'/',
() => true,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
() => 'hello',
'/',
() => true,
() => true,
),
).toBe(false);
});
it('only accepts one or two parts', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b/c',
'/',
() => true,
() => true,
),
).toBe(false);
});
it('checks the prefix and suffix', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => false,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => false,
),
).toBe(false);
});
});
it.each([
[null, true],
[undefined, false],
[1, true],
['a', true],
[() => 'a', false],
[Symbol('a'), false],
[[], true],
[[1], true],
[[undefined], false],
[{}, true],
[{ a: 1 }, true],
[{ a: undefined }, false],
] as [any, boolean][])(`isJsonSafe %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isJsonSafe(value)).toBe(result);
});
it.each([
[null, false],
[7, false],
['', false],
['a', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
['adam.bertil.caesar', true],
['adam.ber-til.caesar', true],
['adam.-bertil.caesar', false],
['adam.bertil-.caesar', false],
['adam/bertil.caesar', false],
[`a.${'b'.repeat(63)}.c`, true],
[`a.${'b'.repeat(64)}.c`, false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(61)}`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(62)}`,
false,
],
])(`isValidDnsSubdomain %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isValidDnsSubdomain(value)).toBe(result);
});
it.each([
[null, false],
[7, false],
['', false],
['a', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
[`${'a'.repeat(63)}`, true],
[`${'a'.repeat(64)}`, false],
])(`isValidDnsLabel %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isValidDnsLabel(value)).toBe(result);
});
it.each([
['', ''],
['a', 'a'],
['a-b', 'ab'],
['-a-b', 'ab'],
['a_b', 'ab'],
[`${'a'.repeat(6000)}`, `${'a'.repeat(6000)}`],
['_:;>!"#€', ''],
])(`normalizeToLowercaseAlphanum %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.normalizeToLowercaseAlphanum(value)).toBe(
result,
);
});
});
plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts
-108
View File
@@ -1,108 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import lodash from 'lodash';
/**
* Contains various helper validation and normalization functions that can be
* composed to form a Validator.
*/
export class CommonValidatorFunctions {
/**
* Checks that the value is on the form <suffix> or <prefix><separator><suffix>, and validates
* those parts separately.
*
* @param value The value to check
* @param separator The separator between parts
* @param isValidPrefix Checks that the part before the separator is valid, if present
* @param isValidSuffix Checks that the part after the separator (or the entire value if there is no separator) is valid
*/
static isValidPrefixAndOrSuffix(
value: any,
separator: string,
isValidPrefix: (value: string) => boolean,
isValidSuffix: (value: string) => boolean,
): boolean {
if (typeof value !== 'string') {
return false;
}
const parts = value.split(separator);
if (parts.length === 1) {
return isValidSuffix(parts[0]);
} else if (parts.length === 2) {
return isValidPrefix(parts[0]) && isValidSuffix(parts[1]);
}
return false;
}
/**
* Checks that the value can be safely transferred as JSON.
*
* @param value The value to check
*/
static isJsonSafe(value: any): boolean {
try {
return lodash.isEqual(value, JSON.parse(JSON.stringify(value)));
} catch {
return false;
}
}
/**
* Checks that the value is a valid DNS subdomain name.
*
* @param value The value to check
* @see https://tools.ietf.org/html/rfc1123
*/
static isValidDnsSubdomain(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 253 &&
value.split('.').every(CommonValidatorFunctions.isValidDnsLabel)
);
}
/**
* Checks that the value is a valid DNS label.
*
* @param value The value to check
* @see https://tools.ietf.org/html/rfc1123
*/
static isValidDnsLabel(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-z0-9]+(\-[a-z0-9]+)*$/.test(value)
);
}
/**
* Normalizes by keeping only a-z, A-Z, and 0-9; and converts to lowercase.
*
* @param value The value to normalize
*/
static normalizeToLowercaseAlphanum(value: string): string {
return value
.split('')
.filter(x => /[a-zA-Z0-9]/.test(x))
.join('')
.toLowerCase();
}
}
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts
-209
View File
@@ -1,209 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions';
describe('KubernetesValidatorFunctions', () => {
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a-b', false],
['a_b', false],
['a.b', false],
['a/a', true],
['a/aAb5C', true],
['a-b.c/v1', true],
['a--b.c/v1', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/v1`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/v1`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidApiVersion %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidApiVersion(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['9AZ', false],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a-b', false],
])(`isValidKind %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidKind(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
])(`isValidObjectName %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', false],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
['a.b', false],
])(`isValidNamespace %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidNamespace(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
['a/a', true],
['a-b.c/a', true],
['a--b.c/a', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/a`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/a`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidLabelKey %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidLabelKey(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', true],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
])(`isValidLabelValue %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
['a/a', true],
['a-b.c/a', true],
['a--b.c/a', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/a`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/a`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidAnnotationKey %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidAnnotationKey(value)).toBe(
matches,
);
});
it.each([
[7, false],
[null, false],
['', true],
['a', true],
['/'.repeat(6000), true],
])(`isValidAnnotationValue %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidAnnotationValue(value)).toBe(
matches,
);
});
});
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts
-86
View File
@@ -1,86 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
/**
* Contains validation functions that match the Kubernetes spec, usable to
* build a catalog that is compatible with those rule sets.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
*/
export class KubernetesValidatorFunctions {
static isValidApiVersion(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
n => n.length >= 1 && n.length <= 63 && /^[a-z0-9A-Z]+$/.test(n),
);
}
static isValidKind(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-zA-Z][a-z0-9A-Z]*$/.test(value)
);
}
static isValidObjectName(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value)
);
}
static isValidNamespace(value: any): boolean {
return CommonValidatorFunctions.isValidDnsLabel(value);
}
static isValidLabelKey(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
KubernetesValidatorFunctions.isValidObjectName,
);
}
static isValidLabelValue(value: any): boolean {
return (
value === '' || KubernetesValidatorFunctions.isValidObjectName(value)
);
}
static isValidAnnotationKey(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
KubernetesValidatorFunctions.isValidObjectName,
);
}
static isValidAnnotationValue(value: any): boolean {
return typeof value === 'string';
}
}
plugins/catalog-backend/src/validation/index.ts
-20
View File
@@ -1,20 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './CommonValidatorFunctions';
export * from './KubernetesValidatorFunctions';
export * from './makeValidator';
export * from './types';
plugins/catalog-backend/src/validation/makeValidator.ts
-38
View File
@@ -1,38 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions';
import { Validators } from './types';
const defaultValidators: Validators = {
isValidApiVersion: KubernetesValidatorFunctions.isValidApiVersion,
isValidKind: KubernetesValidatorFunctions.isValidKind,
isValidEntityName: KubernetesValidatorFunctions.isValidObjectName,
isValidNamespace: KubernetesValidatorFunctions.isValidNamespace,
normalizeEntityName: CommonValidatorFunctions.normalizeToLowercaseAlphanum,
isValidLabelKey: KubernetesValidatorFunctions.isValidLabelKey,
isValidLabelValue: KubernetesValidatorFunctions.isValidLabelValue,
isValidAnnotationKey: KubernetesValidatorFunctions.isValidAnnotationKey,
isValidAnnotationValue: KubernetesValidatorFunctions.isValidAnnotationValue,
};
export function makeValidator(overrides: Partial<Validators> = {}): Validators {
return {
...defaultValidators,
...overrides,
};
}
plugins/catalog-backend/src/validation/types.ts
-27
View File
@@ -1,27 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type Validators = {
isValidApiVersion(value: any): boolean;
isValidKind(value: any): boolean;
isValidEntityName(value: any): boolean;
isValidNamespace(value: any): boolean;
normalizeEntityName(value: string): string;
isValidLabelKey(value: any): boolean;
isValidLabelValue(value: any): boolean;
isValidAnnotationKey(value: any): boolean;
isValidAnnotationValue(value: any): boolean;
};