paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

Merge pull request #1047 from spotify/freben/catalog-model

Browse Source 
Move the model into a plugin-catalog-model for sharing outside the backend
This commit is contained in:
Fredrik Adelöw
2020-05-29 07:02:12 +02:00
committed by GitHub
parent 828e6675c8 06879f5542
commit 273f009297
62 changed files with 1596 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/package.json
+3 -2
View File
@@ -16,8 +16,7 @@
},
"dependencies": {
"@backstage/backend-common": "^0.1.1-alpha.6",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@backstage/catalog-model": "^0.1.1-alpha.6",
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
@@ -38,6 +37,8 @@
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.6",
"@types/lodash": "^4.14.151",
"@types/node-fetch": "^2.5.7",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
"@types/yup": "^0.28.2",
"jest-fetch-mock": "^3.0.3",
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+4 -4
View File
@@ -14,21 +14,21 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { Database } from '../database';
import { DescriptorEnvelope } from '../ingestion/types';
import { EntitiesCatalog, EntityFilters } from './types';
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
constructor(private readonly database: Database) {}
async entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]> {
async entities(filters?: EntityFilters): Promise<Entity[]> {
const items = await this.database.transaction(tx =>
this.database.entities(tx, filters),
);
return items.map(i => i.entity);
}
async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
async entityByUid(uid: string): Promise<Entity | undefined> {
const matches = await this.database.transaction(tx =>
this.database.entities(tx, [{ key: 'uid', values: [uid] }]),
);
@@ -40,7 +40,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
kind: string,
name: string,
namespace: string | undefined,
): Promise<DescriptorEnvelope | undefined> {
): Promise<Entity | undefined> {
const matches = await this.database.transaction(tx =>
this.database.entities(tx, [
{ key: 'kind', values: [kind] },
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.test.ts
+21 -19
View File
@@ -13,13 +13,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import knex from 'knex';
import path from 'path';
import { Database } from '../database';
import { ReaderOutput } from '../ingestion/types';
import { getVoidLogger } from '@backstage/backend-common';
import { IngestionModel } from '../ingestion/types';
import { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
class MockIngestionModel implements IngestionModel {
readLocation = jest.fn(async (type: string, target: string) => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return [{ type: 'data', data: {} as Entity } as const];
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
});
}
describe('DatabaseLocationsCatalog', () => {
const database = knex({
@@ -32,20 +46,7 @@ describe('DatabaseLocationsCatalog', () => {
});
let db: Database;
let catalog: DatabaseLocationsCatalog;
const mockLocationReader = {
read: async (type: string, target: string): Promise<ReaderOutput[]> => {
if (type !== 'valid_type') {
throw new Error(`Unknown location type ${type}`);
}
if (target === 'valid_target') {
return Promise.resolve([{ type: 'data', data: {} }]);
}
throw new Error(
`Can't read location at ${target} with error: Something is broken`,
);
},
};
let ingestionModel: IngestionModel;
beforeEach(async () => {
await database.migrate.latest({
@@ -53,7 +54,8 @@ describe('DatabaseLocationsCatalog', () => {
loadExtensions: ['.ts'],
});
db = new Database(database, getVoidLogger());
catalog = new DatabaseLocationsCatalog(db, mockLocationReader);
ingestionModel = new MockIngestionModel();
catalog = new DatabaseLocationsCatalog(db, ingestionModel);
});
it('resolves to location with id', async () => {
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts
+11 -3
View File
@@ -15,17 +15,25 @@
*/
import { Database } from '../database';
import { IngestionModel } from '../ingestion/types';
import { AddLocation, Location, LocationsCatalog } from './types';
import { LocationReader } from '../ingestion';
export class DatabaseLocationsCatalog implements LocationsCatalog {
constructor(
private readonly database: Database,
private readonly reader: LocationReader,
private readonly ingestionModel: IngestionModel,
) {}
async addLocation(location: AddLocation): Promise<Location> {
const outputs = await this.reader.read(location.type, location.target);
const outputs = await this.ingestionModel.readLocation(
location.type,
location.target,
);
if (!outputs) {
throw new Error(
`Unknown location type ${location.type} ${location.target}`,
);
}
outputs.forEach(output => {
if (output.type === 'error') {
throw new Error(
plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
+6 -6
View File
@@ -15,22 +15,22 @@
*/
import { NotFoundError } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import lodash from 'lodash';
import { DescriptorEnvelope } from '../ingestion';
import { EntitiesCatalog } from './types';
export class StaticEntitiesCatalog implements EntitiesCatalog {
private _entities: DescriptorEnvelope[];
private _entities: Entity[];
constructor(entities: DescriptorEnvelope[]) {
constructor(entities: Entity[]) {
this._entities = entities;
}
async entities(): Promise<DescriptorEnvelope[]> {
async entities(): Promise<Entity[]> {
return lodash.cloneDeep(this._entities);
}
async entityByUid(uid: string): Promise<DescriptorEnvelope | undefined> {
async entityByUid(uid: string): Promise<Entity | undefined> {
const item = this._entities.find(e => uid === e.metadata?.uid);
if (!item) {
throw new NotFoundError('Entity cannot be found');
@@ -42,7 +42,7 @@ export class StaticEntitiesCatalog implements EntitiesCatalog {
kind: string,
name: string,
namespace: string | undefined,
): Promise<DescriptorEnvelope | undefined> {
): Promise<Entity | undefined> {
const item = this._entities.find(
e =>
kind === e.kind &&
plugins/catalog-backend/src/catalog/types.ts
+4 -4
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion';
//
// Entities
@@ -28,13 +28,13 @@ export type EntityFilter = {
export type EntityFilters = EntityFilter[];
export type EntitiesCatalog = {
entities(filters?: EntityFilters): Promise<DescriptorEnvelope[]>;
entityByUid(uid: string): Promise<DescriptorEnvelope | undefined>;
entities(filters?: EntityFilters): Promise<Entity[]>;
entityByUid(uid: string): Promise<Entity | undefined>;
entityByName(
kind: string,
namespace: string | undefined,
name: string,
): Promise<DescriptorEnvelope | undefined>;
): Promise<Entity | undefined>;
};
//
plugins/catalog-backend/src/database/Database.test.ts
+5 -5
View File
@@ -19,9 +19,9 @@ import {
getVoidLogger,
NotFoundError,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import Knex from 'knex';
import path from 'path';
import { DescriptorEnvelope } from '../ingestion';
import { Database } from './Database';
import {
AddDatabaseLocation,
@@ -247,8 +247,8 @@ describe('Database', () => {
describe('entities', () => {
it('can get all entities with empty filters list', async () => {
const catalog = new Database(database, getVoidLogger());
const e1: DescriptorEnvelope = { apiVersion: 'a', kind: 'b' };
const e2: DescriptorEnvelope = {
const e1: Entity = { apiVersion: 'a', kind: 'b' };
const e2: Entity = {
apiVersion: 'a',
kind: 'b',
spec: { c: null },
@@ -271,7 +271,7 @@ describe('Database', () => {
it('can get all specific entities for matching filters (naive case)', async () => {
const catalog = new Database(database, getVoidLogger());
const entities: DescriptorEnvelope[] = [
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b' },
{
apiVersion: 'a',
@@ -305,7 +305,7 @@ describe('Database', () => {
it('can get all specific entities for matching filters with nulls (both missing and literal null value)', async () => {
const catalog = new Database(database, getVoidLogger());
const entities: DescriptorEnvelope[] = [
const entities: Entity[] = [
{ apiVersion: 'a', kind: 'b' },
{
apiVersion: 'a',
plugins/catalog-backend/src/database/Database.ts
+12 -10
View File
@@ -19,12 +19,12 @@ import {
InputError,
NotFoundError,
} from '@backstage/backend-common';
import { Entity, EntityMeta } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntityFilters } from '../catalog';
import { DescriptorEnvelope, EntityMeta } from '../ingestion';
import { buildEntitySearch } from './search';
import {
AddDatabaseLocation,
@@ -54,9 +54,7 @@ function serializeMetadata(metadata: EntityMeta | undefined): string | null {
return JSON.stringify(getStrippedMetadata(metadata));
}
function serializeSpec(
spec: DescriptorEnvelope['spec'],
): DbEntitiesRow['spec'] {
function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] {
if (!spec) {
return null;
}
@@ -66,7 +64,7 @@ function serializeSpec(
function toEntityRow(
locationId: string | undefined,
entity: DescriptorEnvelope,
entity: Entity,
): DbEntitiesRow {
return {
id: entity.metadata!.uid!,
@@ -83,7 +81,7 @@ function toEntityRow(
}
function toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: row.api_version,
kind: row.kind,
metadata: {
@@ -94,7 +92,7 @@ function toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
};
if (row.metadata) {
const metadata = JSON.parse(row.metadata) as DescriptorEnvelope['metadata'];
const metadata = JSON.parse(row.metadata) as Entity['metadata'];
entity.metadata = { ...entity.metadata, ...metadata };
}
@@ -127,7 +125,9 @@ function generateUid(): string {
}
function generateEtag(): string {
return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
return Buffer.from(uuidv4(), 'utf8')
.toString('base64')
.replace(/[^\w]/g, '');
}
/**
@@ -374,7 +374,9 @@ export class Database {
target,
});
return (await tx<DbLocationsRow>('locations').where({ id }).select())![0];
return (await tx<DbLocationsRow>('locations')
.where({ id })
.select())![0];
});
}
@@ -422,7 +424,7 @@ export class Database {
private async updateEntitiesSearch(
tx: Knex.Transaction<any, any>,
entityId: string,
data: DescriptorEnvelope,
data: Entity,
): Promise<void> {
try {
const entries = buildEntitySearch(entityId, data);
plugins/catalog-backend/src/database/DatabaseManager.test.ts
+42 -40
View File
@@ -15,13 +15,9 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import {
ComponentDescriptor,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocationUpdateLogStatus, DbLocationsRow } from './types';
@@ -32,18 +28,18 @@ describe('DatabaseManager', () => {
const db = ({
locations: jest.fn().mockResolvedValue([]),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(),
const reader: IngestionModel = {
readLocation: jest.fn(),
};
const parser: DescriptorParser = {
parse: jest.fn(),
const policy: EntityPolicy = {
enforce: jest.fn(),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).not.toHaveBeenCalled();
expect(parser.parse).not.toHaveBeenCalled();
expect(reader.readLocation).not.toHaveBeenCalled();
expect(policy.enforce).not.toHaveBeenCalled();
});
it('can update a single location', async () => {
@@ -52,7 +48,7 @@ describe('DatabaseManager', () => {
type: 'some',
target: 'thing',
};
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
@@ -68,18 +64,20 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as Partial<Database>) as Database;
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.read).toHaveBeenCalledTimes(1);
expect(reader.read).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(reader.readLocation).toHaveBeenCalledTimes(1);
expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(db.addEntity).toHaveBeenCalledTimes(1);
expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, {
locationId: '123',
@@ -108,21 +106,23 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() => Promise.resolve(desc)),
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -158,23 +158,25 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: ComponentDescriptor = {
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: LocationReader = {
read: jest.fn(() => Promise.resolve([{ type: 'data', data: desc }])),
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
@@ -211,19 +213,19 @@ describe('DatabaseManager', () => {
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const reader: LocationReader = {
read: jest.fn(() =>
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.reject([{ type: 'error', error: new Error('test message') }]),
),
};
const parser: DescriptorParser = {
parse: jest.fn(() =>
Promise.reject(new ParserError('parser error message', 'c1')),
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, parser, getVoidLogger()),
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
plugins/catalog-backend/src/database/DatabaseManager.ts
+12 -20
View File
@@ -14,16 +14,12 @@
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import path from 'path';
import { Logger } from 'winston';
import {
DescriptorEnvelope,
DescriptorParser,
LocationReader,
ParserError,
} from '../ingestion';
import { IngestionModel } from '../ingestion/types';
import { Database } from './Database';
import { DatabaseLocationUpdateLogStatus, DbEntityRequest } from './types';
@@ -67,8 +63,8 @@ export class DatabaseManager {
public static async refreshLocations(
database: Database,
reader: LocationReader,
parser: DescriptorParser,
ingestionModel: IngestionModel,
entityPolicy: EntityPolicy,
logger: Logger,
): Promise<void> {
const locations = await database.locations();
@@ -78,7 +74,10 @@ export class DatabaseManager {
`Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
);
const readerOutput = await reader.read(location.type, location.target);
const readerOutput = await ingestionModel.readLocation(
location.type,
location.target,
);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
@@ -87,7 +86,7 @@ export class DatabaseManager {
}
try {
const entity = await parser.parse(readerItem.data);
const entity = await entityPolicy.enforce(readerItem.data);
await DatabaseManager.refreshSingleEntity(
database,
location.id,
@@ -100,15 +99,11 @@ export class DatabaseManager {
entity.metadata!.name,
);
} catch (error) {
let entityName;
if (error instanceof ParserError) {
entityName = error.entityName;
}
await DatabaseManager.logUpdateFailure(
database,
location.id,
error,
entityName,
readerItem.data.metadata?.name,
);
}
}
@@ -129,7 +124,7 @@ export class DatabaseManager {
private static async refreshSingleEntity(
database: Database,
locationId: string,
entity: DescriptorEnvelope,
entity: Entity,
logger: Logger,
): Promise<void> {
const { kind } = entity;
@@ -163,10 +158,7 @@ export class DatabaseManager {
});
}
private static entitiesAreEqual(
first: DescriptorEnvelope,
second: DescriptorEnvelope,
) {
private static entitiesAreEqual(first: Entity, second: Entity) {
const firstClone = lodash.cloneDeep(first);
const secondClone = lodash.cloneDeep(second);
plugins/catalog-backend/src/database/migrations/20200520140700_location_update_log_table.ts
+4 -1
View File
@@ -19,7 +19,10 @@ export async function up(knex: Knex): Promise<any> {
return knex.schema.createTable('location_update_log', table => {
table.uuid('id').primary();
table.enum('status', ['success', 'fail']).notNullable();
table.dateTime('created_at').defaultTo(knex.fn.now()).notNullable();
table
.dateTime('created_at')
.defaultTo(knex.fn.now())
.notNullable();
table.string('message');
table
.uuid('location_id')
plugins/catalog-backend/src/database/search.test.ts
+3 -3
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { DescriptorEnvelope } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { buildEntitySearch, visitEntityPart } from './search';
import { DbEntitiesSearchRow } from './types';
@@ -99,7 +99,7 @@ describe('search', () => {
describe('buildEntitySearch', () => {
it('adds special keys even if missing', () => {
const input: DescriptorEnvelope = {
const input: Entity = {
apiVersion: 'a',
kind: 'b',
};
@@ -116,7 +116,7 @@ describe('search', () => {
});
it('adds prefix-stripped versions', () => {
const input: DescriptorEnvelope = {
const input: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
plugins/catalog-backend/src/database/search.ts
+2 -2
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { DescriptorEnvelope } from '../ingestion';
import { Entity } from '@backstage/catalog-model';
import { DbEntitiesSearchRow } from './types';
// Search entries that start with these prefixes, also get a shorthand without
@@ -119,7 +119,7 @@ export function visitEntityPart(
*/
export function buildEntitySearch(
entityId: string,
entity: DescriptorEnvelope,
entity: Entity,
): DbEntitiesSearchRow[] {
// Start with some special keys that are always present because you want to
// be able to easily search for null specifically
plugins/catalog-backend/src/database/types.ts
+3 -3
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import * as yup from 'yup';
import { DescriptorEnvelope } from '../ingestion';
export type DbEntitiesRow = {
id: string;
@@ -32,12 +32,12 @@ export type DbEntitiesRow = {
export type DbEntityRequest = {
locationId?: string;
entity: DescriptorEnvelope;
entity: Entity;
};
export type DbEntityResponse = {
locationId?: string;
entity: DescriptorEnvelope;
entity: Entity;
};
export type DbEntitiesSearchRow = {
plugins/catalog-backend/src/ingestion/DescriptorParsers.ts
-53
View File
@@ -1,53 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { makeValidator } from '../validation';
import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser';
import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser';
import {
DescriptorEnvelope,
DescriptorParser,
KindParser,
ParserError,
} from './types';
export class DescriptorParsers implements DescriptorParser {
static create(): DescriptorParser {
const validators = makeValidator();
return new DescriptorParsers(new DescriptorEnvelopeParser(validators), [
new ComponentDescriptorV1beta1Parser(),
]);
}
constructor(
private readonly envelopeParser: DescriptorEnvelopeParser,
private readonly kindParsers: KindParser[],
) {}
async parse(descriptor: object): Promise<DescriptorEnvelope> {
const envelope = await this.envelopeParser.parse(descriptor);
for (const parser of this.kindParsers) {
const parsed = await parser.tryParse(envelope);
if (parsed) {
return parsed;
}
}
throw new ParserError(
`Unsupported object ${envelope.apiVersion}, ${envelope.kind}`,
envelope.metadata?.name,
);
}
}
plugins/catalog-backend/src/ingestion/IngestionModels.ts
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicy, EntityPolicies } from '@backstage/catalog-model';
import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types';
import { LocationReader, LocationReaders } from './source';
import { IngestionModel } from './types';
import { DescriptorParsers } from './descriptor';
export class IngestionModels implements IngestionModel {
private readonly reader: LocationReader;
private readonly parser: DescriptorParser;
private readonly entityPolicy: EntityPolicy;
static default(): IngestionModel {
return new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
}
constructor(
reader: LocationReader,
parser: DescriptorParser,
entityPolicy: EntityPolicy,
) {
this.reader = reader;
this.parser = parser;
this.entityPolicy = entityPolicy;
}
async readLocation(type: string, target: string): Promise<ReaderOutput[]> {
const buffer = await this.reader.tryRead(type, target);
if (!buffer) {
throw new Error(`No reader could handle location ${type} ${target}`);
}
const items = await this.parser.tryParse(buffer);
if (!items) {
throw new Error(`No parser could handle location ${type} ${target}`);
}
const result: ReaderOutput[] = [];
for (const item of items) {
if (item.type === 'error') {
result.push(item);
} else {
try {
const output = await this.entityPolicy.enforce(item.data);
result.push({ type: 'data', data: output });
} catch (e) {
result.push({ type: 'error', error: e });
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/LocationReaders.ts
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationSource } from './sources/FileLocationSource';
import { GitHubLocationSource } from './sources/GitHubLocationSource';
import { LocationReader, LocationSource, ReaderOutput } from './types';
export class LocationReaders implements LocationReader {
static create(): LocationReader {
return new LocationReaders({
file: new FileLocationSource(),
github: new GitHubLocationSource(),
});
}
constructor(private readonly sources: Record<string, LocationSource>) {}
async read(type: string, target: string): Promise<ReaderOutput[]> {
const source = this.sources[type];
if (!source) {
throw new Error(`Unknown location type ${type}`);
}
return source.read(target);
}
}
plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DescriptorParser, ReaderOutput } from './parsers/types';
import { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
/**
* Parses raw descriptor data (e.g. from a file or stream) into entities.
*/
export class DescriptorParsers implements DescriptorParser {
private readonly parsers: DescriptorParser[];
static defaultParsers(): DescriptorParser[] {
return [new YamlDescriptorParser()];
}
constructor(
parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(),
) {
this.parsers = parsers;
}
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
for (const parser of this.parsers) {
const result = await parser.tryParse(data);
if (result) {
return result;
}
}
throw new Error(`Unsupported descriptor format`);
}
}
plugins/catalog-backend/src/validation/index.ts → plugins/catalog-backend/src/ingestion/descriptor/index.ts
+2 -4
View File
@@ -14,7 +14,5 @@
* limitations under the License.
*/
export * from './CommonValidatorFunctions';
export * from './KubernetesValidatorFunctions';
export * from './makeValidator';
export * from './types';
export { DescriptorParsers } from './DescriptorParsers';
export { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts
+64
View File
@@ -0,0 +1,64 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import yaml from 'yaml';
import { DescriptorParser, ReaderOutput } from './types';
/**
* Parses descriptors on YAML format
*/
export class YamlDescriptorParser implements DescriptorParser {
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
// TODO(freben): Should perhaps first do format detection, so the parse
// failure can be emitted as a proper error instead of just as if we
// weren't handling the format at all.
let documents;
try {
documents = yaml.parseAllDocuments(data.toString('utf8'));
} catch (e) {
return undefined;
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json as Entity,
});
}
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: Entity };
/**
* Parses raw descriptor data (e.g. from a file) into entities.
*/
export type DescriptorParser = {
/**
* Try to parse some raw data into an entity.
*
* Note that this is only the low level operation of parsing the raw file
* format, e.g. reading JSON or YAML or similar and emitting as structured
* but unvalidated data. The actual validation is performed by EntityPolicy
* and KindParser.
*
* @param data Raw descriptor data
* @returns A list of raw unvalidated entities / errors, or undefined if the
* given data is not meant to be handled by this parser
* @throws An Error if the format was handled and found to not be properly
* formed
*/
tryParse(data: Buffer): Promise<ReaderOutput[] | undefined>;
};
plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts
-63
View File
@@ -1,63 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { DescriptorEnvelope, KindParser, ParserError } from '../types';
export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope {
spec: {
type: string;
};
}
export class ComponentDescriptorV1beta1Parser implements KindParser {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<ComponentDescriptorV1beta1>>({
metadata: yup
.object({
name: yup.string().required(),
})
.required(),
spec: yup
.object({
type: yup.string().required(),
})
.required(),
});
}
async tryParse(
envelope: DescriptorEnvelope,
): Promise<DescriptorEnvelope | undefined> {
if (
envelope.apiVersion !== 'backstage.io/v1beta1' ||
envelope.kind !== 'Component'
) {
return undefined;
}
try {
return await this.schema.validate(envelope, { strict: true });
} catch (e) {
throw new ParserError(
`Malformed component, ${e}`,
envelope.metadata?.name,
);
}
}
}
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts
-172
View File
@@ -1,172 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { makeValidator } from '../../validation';
import { DescriptorEnvelopeParser } from './DescriptorEnvelopeParser';
describe('DescriptorEnvelopeParser', () => {
let data: any;
let parser: DescriptorEnvelopeParser;
beforeEach(() => {
data = yaml.parse(`
apiVersion: backstage.io/v1beta1
kind: Component
metadata:
uid: e01199ab-08cc-44c2-8e19-5c29ded82521
etag: lsndfkjsndfkjnsdfkjnsd==
generation: 13
name: my-component-yay
namespace: the-namespace
labels:
backstage.io/custom: ValueStuff
annotations:
example.com/bindings: are-secret
spec:
custom: stuff
`);
parser = new DescriptorEnvelopeParser(makeValidator());
});
it('works for the happy path', async () => {
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects missing apiVersion', async () => {
delete data.apiVersion;
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects wrong root type', async () => {
await expect(parser.parse(7)).rejects.toThrow(/object/);
});
it('rejects bad apiVersion', async () => {
data.apiVersion = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/apiVersion/);
});
it('rejects missing kind', async () => {
delete data.kind;
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('rejects bad kind', async () => {
data.kind = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/kind/);
});
it('accepts missing metadata', async () => {
delete data.medatata;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object metadata', async () => {
data.metadata = 7;
await expect(parser.parse(data)).rejects.toThrow(/metadata/);
});
it('accepts missing uid', async () => {
delete data.metadata.uid;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad uid', async () => {
data.metadata.uid = 7;
await expect(parser.parse(data)).rejects.toThrow(/uid/);
});
it('accepts missing etag', async () => {
delete data.metadata.etag;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad etag', async () => {
data.metadata.etag = 7;
await expect(parser.parse(data)).rejects.toThrow(/etag/);
});
it('accepts missing generation', async () => {
delete data.metadata.generation;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects bad generation', async () => {
data.metadata.generation = 'a';
await expect(parser.parse(data)).rejects.toThrow(/generation/);
});
it('accepts missing spec', async () => {
delete data.spec;
await expect(parser.parse(data)).resolves.toBe(data);
});
it('rejects non-object spec', async () => {
data.spec = 7;
await expect(parser.parse(data)).rejects.toThrow(/spec/);
});
it('rejects bad name', async () => {
data.metadata.name = 7;
await expect(parser.parse(data)).rejects.toThrow(/name/);
});
it('rejects bad namespace', async () => {
data.metadata.namespace = 7;
await expect(parser.parse(data)).rejects.toThrow(/namespace/);
});
it('rejects bad label key', async () => {
data.metadata.labels['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/label.*key/i);
});
it('rejects bad label value', async () => {
data.metadata.labels.a = 'a#b';
await expect(parser.parse(data)).rejects.toThrow(/label.*value/i);
});
it('rejects bad annotation key', async () => {
data.metadata.annotations['a#b'] = 'value';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*key/i);
});
it('rejects bad annotation value', async () => {
data.metadata.annotations.a = [];
await expect(parser.parse(data)).rejects.toThrow(/annotation.*value/i);
});
it('rejects unknown root keys', async () => {
data.spec2 = {};
await expect(parser.parse(data)).rejects.toThrow(/spec2/i);
});
it('rejects reserved keys in the spec root', async () => {
data.spec.apiVersion = 'a/b';
await expect(parser.parse(data)).rejects.toThrow(/spec.*apiVersion/i);
});
it('rejects reserved keys in labels', async () => {
data.metadata.labels.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/label.*apiVersion/i);
});
it('rejects reserved keys in annotations', async () => {
data.metadata.annotations.apiVersion = 'a';
await expect(parser.parse(data)).rejects.toThrow(/annotation.*apiVersion/i);
});
});
plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts
-206
View File
@@ -1,206 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { Validators } from '../../validation';
import { DescriptorEnvelope } from '../types';
/**
* Parses some raw structured data as a descriptor envelope
*/
export class DescriptorEnvelopeParser {
private schema: yup.Schema<DescriptorEnvelope>;
constructor(validators: Validators) {
const apiVersionSchema = yup
.string()
.required()
.test(
'apiVersion',
'The apiVersion is not formatted according to schema',
validators.isValidApiVersion,
);
const kindSchema = yup
.string()
.required()
.test(
'kind',
'The kind is not formatted according to schema',
validators.isValidKind,
);
const uidSchema = yup
.string()
.notRequired()
.test(
'metadata.uid',
'The uid is not formatted according to schema',
value => value === undefined || value.length > 0,
);
const etagSchema = yup
.string()
.notRequired()
.test(
'metadata.etag',
'The etag value is not according to schema',
value => value === undefined || value.length > 0,
);
const generationSchema = yup
.number()
.notRequired()
.test(
'metadata.generation',
'The generation value is not according to schema',
value => value === undefined || value > 0,
);
const nameSchema = yup
.string()
.notRequired()
.test(
'metadata.name',
'The name is not formatted according to schema',
value => value === undefined || validators.isValidEntityName(value),
);
const namespaceSchema = yup
.string()
.notRequired()
.test(
'metadata.namespace',
'The namespace is malformed',
value => value === undefined || validators.isValidNamespace(value),
);
const labelsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.labels.keys',
message: 'Label keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidLabelKey)
);
},
})
.test({
name: 'metadata.labels.values',
message: 'Label values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidLabelValue)
);
},
});
const annotationsSchema = yup
.object<Record<string, string>>()
.notRequired()
.test({
name: 'metadata.annotations.keys',
message: 'Annotation keys not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.keys(value).every(validators.isValidAnnotationKey)
);
},
})
.test({
name: 'metadata.annotations.values',
message: 'Annotation values not formatted according to schema',
test(value: object) {
return (
value === undefined ||
Object.values(value).every(validators.isValidAnnotationValue)
);
},
});
const metadataSchema = yup
.object({
uid: uidSchema,
etag: etagSchema,
generation: generationSchema,
name: nameSchema,
namespace: namespaceSchema,
labels: labelsSchema,
annotations: annotationsSchema,
})
.notRequired();
const specSchema = yup.object({}).notRequired();
this.schema = yup
.object({
apiVersion: apiVersionSchema,
kind: kindSchema,
metadata: metadataSchema,
spec: specSchema,
})
.noUnknown();
}
async parse(data: any): Promise<DescriptorEnvelope> {
let result: DescriptorEnvelope;
try {
result = await this.schema.validate(data, { strict: true });
} catch (e) {
throw new Error(`Malformed envelope, ${e}`);
}
// These are keys with specific semantic meaning in a document, that we do
// not want to appear in the root of the spec, or as labels or as
// annotations, because they will lead to confusion.
const reservedKeys = [
'apiVersion',
'kind',
'uid',
'etag',
'generation',
'name',
'namespace',
'labels',
'annotations',
'spec',
];
for (const key of reservedKeys) {
if (result.spec?.hasOwnProperty(key)) {
throw new Error(
`The spec may not contain the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.labels?.hasOwnProperty(key)) {
throw new Error(
`A label may not have the key ${key}, because it has reserved meaning`,
);
}
if (result.metadata?.annotations?.hasOwnProperty(key)) {
throw new Error(
`An annotation may not have the key ${key}, because it has reserved meaning`,
);
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/index.ts
+4 -3
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export * from './DescriptorParsers';
export * from './LocationReaders';
export * from './types';
export * from './descriptor';
export { IngestionModels } from './IngestionModels';
export * from './source';
export type { IngestionModel } from './types';
plugins/catalog-backend/src/ingestion/source/LocationReaders.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationReader } from './readers/FileLocationReader';
import { GitHubLocationReader } from './readers/GitHubLocationReader';
import { LocationReader } from './readers/types';
export class LocationReaders implements LocationReader {
private readonly readers: LocationReader[];
static defaultReaders(): LocationReader[] {
return [new FileLocationReader(), new GitHubLocationReader()];
}
constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) {
this.readers = readers;
}
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
for (const reader of this.readers) {
const result = await reader.tryRead(type, target);
if (result) {
return result;
}
}
throw new Error(`Could not read unknown location "${type}", "${target}"`);
}
}
plugins/catalog-backend/src/validation/types.ts → plugins/catalog-backend/src/ingestion/source/index.ts
+4 -11
View File
@@ -14,14 +14,7 @@
* limitations under the License.
*/
export type Validators = {
isValidApiVersion(value: any): boolean;
isValidKind(value: any): boolean;
isValidEntityName(value: any): boolean;
isValidNamespace(value: any): boolean;
normalizeEntityName(value: string): string;
isValidLabelKey(value: any): boolean;
isValidLabelValue(value: any): boolean;
isValidAnnotationKey(value: any): boolean;
isValidAnnotationValue(value: any): boolean;
};
export { LocationReaders } from './LocationReaders';
export { FileLocationReader } from './readers/FileLocationReader';
export { GitHubLocationReader } from './readers/GitHubLocationReader';
export type { LocationReader } from './readers/types';
plugins/catalog-backend/src/ingestion/sources/FileLocationSource.ts → plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts
+11 -12
View File
@@ -15,22 +15,21 @@
*/
import fs from 'fs-extra';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
import { LocationReader } from './types';
/**
* Reads a file from the local file system.
*/
export class FileLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'file') {
return undefined;
}
export class FileLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let rawYaml;
try {
rawYaml = await fs.readFile(target, 'utf8');
return await fs.readFile(target);
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/sources/__tests__/GitHubLocationSource.test.ts → plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts
+17 -35
View File
@@ -16,43 +16,26 @@
jest.mock('node-fetch');
import fs from 'fs-extra';
import fetch from 'node-fetch';
import path from 'path';
import { GitHubLocationSource } from '../GitHubLocationSource';
import { GitHubLocationReader } from './GitHubLocationReader';
const { Response } = jest.requireActual('node-fetch');
const FIXTURES_DIR = path.resolve(
__dirname,
'..',
'..',
'..',
'..',
'fixtures',
);
const fixtures = fs.readdirSync(FIXTURES_DIR).reduce((acc, filename) => {
acc[filename] = fs.readFileSync(path.resolve(FIXTURES_DIR, filename), 'utf8');
return acc;
}, {} as Record<string, string>);
describe('Unit: GitHubLocationSource', () => {
describe('Unit: GitHubLocationReader', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('fetches the file and parses it correctly', async () => {
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures['one_component.yaml'])),
);
const reader = new GitHubLocationSource();
(fetch as any).mockResolvedValueOnce(new Response('hello'));
const result = await reader.read(
const reader = new GitHubLocationReader();
const buffer = await reader.tryRead(
'github',
'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
expect(buffer?.toString('utf8')).toBe('hello');
});
it('changes the url to point to https://raw.githubusercontent.com', async () => {
@@ -61,12 +44,12 @@ describe('Unit: GitHubLocationSource', () => {
const folderPath = `master/plugins/catalog-backend/fixtures`;
const componentFilename = `one_component.yaml`;
const rawGitHubUrl = `https://raw.githubusercontent.com`;
const reader = new GitHubLocationSource();
(fetch as any).mockReturnValueOnce(
Promise.resolve(new Response(fixtures[componentFilename])),
);
await reader.read(
const reader = new GitHubLocationReader();
(fetch as any).mockResolvedValueOnce(new Response('hello'));
await reader.tryRead(
'github',
`${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
);
@@ -76,7 +59,7 @@ describe('Unit: GitHubLocationSource', () => {
});
describe('rejects wrong urls', () => {
const reader = new GitHubLocationSource();
const reader = new GitHubLocationReader();
it.each([
['http://example.com/one_component.yaml'],
@@ -87,7 +70,7 @@ describe('Unit: GitHubLocationSource', () => {
])(
'%p',
async (url: string) =>
await expect(reader.read(url)).rejects.toThrow(/url/),
await expect(reader.tryRead('github', url)).rejects.toThrow(/url/),
);
});
});
@@ -100,11 +83,10 @@ describe('Integration: GitHubLocationSource', () => {
it('fetches the fixture from backstage repo', async () => {
const PERMANENT_LINK =
'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
const reader = new GitHubLocationSource();
const result = await reader.read(PERMANENT_LINK);
const reader = new GitHubLocationReader();
const result = await reader.tryRead('github', PERMANENT_LINK);
expect(result[0].type).toBe('data');
expect((result[0] as any).data.metadata.name).toBe('component3');
expect(result?.toString('utf8')).toContain('component3');
});
});
plugins/catalog-backend/src/ingestion/sources/GitHubLocationSource.ts → plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts
+24 -23
View File
@@ -16,17 +16,31 @@
import fetch from 'node-fetch';
import { URL } from 'url';
import { LocationSource, ReaderOutput } from '../types';
import { readDescriptorYaml } from './util';
import { LocationReader } from './types';
// Pointing to raw.githubusercontent.com for now
// to be changed in the future, after auth and tokens are done
export class GitHubLocationSource implements LocationSource {
async read(target: string): Promise<ReaderOutput[]> {
let url: URL;
/**
* Reads a file whose target is a GitHub URL.
*
* Uses raw.githubusercontent.com for now, but this will probably change in the
* future when token auth is implemented.
*/
export class GitHubLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'github') {
return undefined;
}
const url = this.buildRawUrl(target);
try {
url = new URL(target);
return await fetch(url.toString()).then(x => x.buffer());
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
}
private buildRawUrl(target: string): URL {
try {
const url = new URL(target);
const [
empty,
@@ -51,23 +65,10 @@ export class GitHubLocationSource implements LocationSource {
url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/');
url.hostname = 'raw.githubusercontent.com';
url.protocol = 'https';
return url;
} catch (e) {
throw new Error(`Incorrect url: ${target}, ${e}`);
}
let rawYaml;
try {
rawYaml = await fetch(url.toString()).then(x => {
return x.text();
});
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
try {
return readDescriptorYaml(rawYaml);
} catch (e) {
throw new Error(`Malformed descriptor at "${target}", ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/source/readers/types.ts
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The target contents, as a raw Buffer, or undefined if this type
* was not meant to be consumed by this reader
* @throws An error if the type was meant for this reader, but could not be
* read
*/
tryRead(type: string, target: string): Promise<Buffer | undefined>;
};
plugins/catalog-backend/src/ingestion/sources/util.ts
-55
View File
@@ -1,55 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yaml from 'yaml';
import { ReaderOutput } from '../types';
export function readDescriptorYaml(data: string): ReaderOutput[] {
let documents;
try {
documents = yaml.parseAllDocuments(data);
} catch (e) {
throw new Error(`Could not parse YAML data, ${e}`);
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json,
});
}
}
}
}
return result;
}
plugins/catalog-backend/src/ingestion/types.ts
+3 -167
View File
@@ -14,172 +14,8 @@
* limitations under the License.
*/
import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser';
import { ReaderOutput } from './descriptor/parsers/types';
export type ComponentDescriptor = ComponentDescriptorV1beta1;
/**
* Metadata fields common to all versions/kinds of entity.
*
* @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
*/
export type EntityMeta = {
/**
* A globally unique ID for the entity.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, but the server is free to reject requests
* that do so in such a way that it breaks semantics.
*/
uid?: string;
/**
* An opaque string that changes for each update operation to any part of
* the entity, including metadata.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations. The field can (optionally) be specified when performing
* update or delete operations, and the server will then reject the
* operation if it does not match the current stored value.
*/
etag?: string;
/**
* A positive nonzero number that indicates the current generation of data
* for this entity; the value is incremented each time the spec changes.
*
* This field can not be set by the user at creation time, and the server
* will reject an attempt to do so. The field will be populated in read
* operations.
*/
generation?: number;
/**
* The name of the entity.
*
* Must be uniqe within the catalog at any given point in time, for any
* given namespace, for any given kind.
*/
name?: string;
/**
* The namespace that the entity belongs to.
*/
namespace?: string;
/**
* Key/value pairs of identifying information attached to the entity.
*/
labels?: Record<string, string>;
/**
* Key/value pairs of non-identifying auxiliary information attached to the
* entity.
*/
annotations?: Record<string, string>;
};
/**
* The format envelope that's common to all versions/kinds.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
*/
export type DescriptorEnvelope = {
/**
* The version of specification format for this particular entity that
* this is written against.
*/
apiVersion: string;
/**
* The high level entity type being described.
*/
kind: string;
/**
* Optional metadata related to the entity.
*/
metadata?: EntityMeta;
/**
* The specification data describing the entity itself.
*/
spec?: object;
};
/**
* Parses and validates descriptors.
*
* The output must be validated and well formed.
*/
export type DescriptorParser = {
/**
* Parses and validates a single raw descriptor.
*
* @param descriptor A raw descriptor object
* @returns A structure describing the parsed and validated descriptor
* @throws An Error if the descriptor was malformed
*/
parse(descriptor: object): Promise<DescriptorEnvelope>;
};
/**
* Parses and validates a single envelope into its materialized kind.
*
* These parsers may assume that the envelope is already validated and well
* formed.
*/
export type KindParser = {
/**
* Try to parse an envelope into a materialized kind.
*
* @param envelope A valid descriptor envelope
* @returns A materialized type, or undefined if the given version/kind is
* not meant to be handled by this parser
* @throws An Error if the type was handled and found to not be properly
* formatted
*/
tryParse(
envelope: DescriptorEnvelope,
): Promise<DescriptorEnvelope | undefined>;
};
export class ParserError extends Error {
constructor(message?: string, private _entityName?: string | undefined) {
super(message);
}
get entityName() {
return this._entityName;
}
}
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: object };
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The parsed contents, as an array of unverified descriptors or
* errors where the individual documents could not be parsed.
* @throws An error if the location as a whole could not be read
*/
read(type: string, target: string): Promise<ReaderOutput[]>;
};
export type LocationSource = {
/**
* Reads the contents of a single location.
*
* @param target The location target to read
* @returns The parsed contents, as an array of unverified descriptors
* @throws An error if the location target could not be read
*/
read(target: string): Promise<ReaderOutput[]>;
export type IngestionModel = {
readLocation(type: string, target: string): Promise<ReaderOutput[]>;
};
plugins/catalog-backend/src/service/router.test.ts
+7 -5
View File
@@ -15,10 +15,10 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import request from 'supertest';
import { EntitiesCatalog, Location, LocationsCatalog } from '../catalog';
import { DescriptorEnvelope } from '../ingestion';
import { createRouter } from './router';
class MockEntitiesCatalog implements EntitiesCatalog {
@@ -37,7 +37,7 @@ class MockLocationsCatalog implements LocationsCatalog {
describe('createRouter', () => {
describe('entities', () => {
it('happy path: lists entities', async () => {
const entities: DescriptorEnvelope[] = [{ apiVersion: 'a', kind: 'b' }];
const entities: Entity[] = [{ apiVersion: 'a', kind: 'b' }];
const catalog = new MockEntitiesCatalog();
catalog.entities.mockResolvedValueOnce(entities);
@@ -76,7 +76,7 @@ describe('createRouter', () => {
describe('entityByUid', () => {
it('can fetch entity by uid', async () => {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
@@ -117,7 +117,7 @@ describe('createRouter', () => {
describe('entityByName', () => {
it('can fetch entity by name', async () => {
const entity: DescriptorEnvelope = {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
@@ -190,7 +190,9 @@ describe('createRouter', () => {
});
const app = express().use(router);
const response = await request(app).post('/locations').send(location);
const response = await request(app)
.post('/locations')
.send(location);
expect(response.status).toEqual(400);
});
plugins/catalog-backend/src/service/router.ts
+1 -1
View File
@@ -22,7 +22,7 @@ import {
addLocationSchema,
EntitiesCatalog,
EntityFilters,
LocationsCatalog,
LocationsCatalog
} from '../catalog';
import { validateRequestBody } from './util';
plugins/catalog-backend/src/validation/CommonValidatorFunctions.test.ts
-178
View File
@@ -1,178 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
describe('CommonValidatorFunctions', () => {
describe('isValidPrefixAndOrSuffix', () => {
it('only accepts strings', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
null,
'/',
() => true,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
7,
'/',
() => true,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
() => 'hello',
'/',
() => true,
() => true,
),
).toBe(false);
});
it('only accepts one or two parts', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b/c',
'/',
() => true,
() => true,
),
).toBe(false);
});
it('checks the prefix and suffix', () => {
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => true,
),
).toBe(true);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => false,
() => true,
),
).toBe(false);
expect(
CommonValidatorFunctions.isValidPrefixAndOrSuffix(
'a/b',
'/',
() => true,
() => false,
),
).toBe(false);
});
});
it.each([
[null, true],
[undefined, false],
[1, true],
['a', true],
[() => 'a', false],
[Symbol('a'), false],
[[], true],
[[1], true],
[[undefined], false],
[{}, true],
[{ a: 1 }, true],
[{ a: undefined }, false],
] as [any, boolean][])(`isJsonSafe %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isJsonSafe(value)).toBe(result);
});
it.each([
[null, false],
[7, false],
['', false],
['a', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
['adam.bertil.caesar', true],
['adam.ber-til.caesar', true],
['adam.-bertil.caesar', false],
['adam.bertil-.caesar', false],
['adam/bertil.caesar', false],
[`a.${'b'.repeat(63)}.c`, true],
[`a.${'b'.repeat(64)}.c`, false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(61)}`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(62)}`,
false,
],
])(`isValidDnsSubdomain %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isValidDnsSubdomain(value)).toBe(result);
});
it.each([
[null, false],
[7, false],
['', false],
['a', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
[`${'a'.repeat(63)}`, true],
[`${'a'.repeat(64)}`, false],
])(`isValidDnsLabel %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.isValidDnsLabel(value)).toBe(result);
});
it.each([
['', ''],
['a', 'a'],
['a-b', 'ab'],
['-a-b', 'ab'],
['a_b', 'ab'],
[`${'a'.repeat(6000)}`, `${'a'.repeat(6000)}`],
['_:;>!"#€', ''],
])(`normalizeToLowercaseAlphanum %p ? %p`, (value, result) => {
expect(CommonValidatorFunctions.normalizeToLowercaseAlphanum(value)).toBe(
result,
);
});
});
plugins/catalog-backend/src/validation/CommonValidatorFunctions.ts
-108
View File
@@ -1,108 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import lodash from 'lodash';
/**
* Contains various helper validation and normalization functions that can be
* composed to form a Validator.
*/
export class CommonValidatorFunctions {
/**
* Checks that the value is on the form <suffix> or <prefix><separator><suffix>, and validates
* those parts separately.
*
* @param value The value to check
* @param separator The separator between parts
* @param isValidPrefix Checks that the part before the separator is valid, if present
* @param isValidSuffix Checks that the part after the separator (or the entire value if there is no separator) is valid
*/
static isValidPrefixAndOrSuffix(
value: any,
separator: string,
isValidPrefix: (value: string) => boolean,
isValidSuffix: (value: string) => boolean,
): boolean {
if (typeof value !== 'string') {
return false;
}
const parts = value.split(separator);
if (parts.length === 1) {
return isValidSuffix(parts[0]);
} else if (parts.length === 2) {
return isValidPrefix(parts[0]) && isValidSuffix(parts[1]);
}
return false;
}
/**
* Checks that the value can be safely transferred as JSON.
*
* @param value The value to check
*/
static isJsonSafe(value: any): boolean {
try {
return lodash.isEqual(value, JSON.parse(JSON.stringify(value)));
} catch {
return false;
}
}
/**
* Checks that the value is a valid DNS subdomain name.
*
* @param value The value to check
* @see https://tools.ietf.org/html/rfc1123
*/
static isValidDnsSubdomain(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 253 &&
value.split('.').every(CommonValidatorFunctions.isValidDnsLabel)
);
}
/**
* Checks that the value is a valid DNS label.
*
* @param value The value to check
* @see https://tools.ietf.org/html/rfc1123
*/
static isValidDnsLabel(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-z0-9]+(\-[a-z0-9]+)*$/.test(value)
);
}
/**
* Normalizes by keeping only a-z, A-Z, and 0-9; and converts to lowercase.
*
* @param value The value to normalize
*/
static normalizeToLowercaseAlphanum(value: string): string {
return value
.split('')
.filter(x => /[a-zA-Z0-9]/.test(x))
.join('')
.toLowerCase();
}
}
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.test.ts
-209
View File
@@ -1,209 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions';
describe('KubernetesValidatorFunctions', () => {
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a-b', false],
['a_b', false],
['a.b', false],
['a/a', true],
['a/aAb5C', true],
['a-b.c/v1', true],
['a--b.c/v1', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/v1`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/v1`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidApiVersion %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidApiVersion(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['9AZ', false],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a-b', false],
])(`isValidKind %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidKind(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
])(`isValidObjectName %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidObjectName(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', false],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', false],
['a.b', false],
])(`isValidNamespace %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidNamespace(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
['a/a', true],
['a-b.c/a', true],
['a--b.c/a', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/a`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/a`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidLabelKey %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidLabelKey(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', true],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', false],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
])(`isValidLabelValue %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidLabelValue(value)).toBe(matches);
});
it.each([
[7, false],
[null, false],
['', false],
['a', true],
['AZ09', true],
['a'.repeat(63), true],
['a'.repeat(64), false],
['a/b', true],
['a-b', true],
['-a-b', false],
['a-b-', false],
['a--b', false],
['a_b', true],
['a.b', true],
['a/a', true],
['a-b.c/a', true],
['a--b.c/a', false],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
61,
)}/a`,
true,
],
[
`${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(63)}.${'a'.repeat(
62,
)}/a`,
false,
],
[`a/${'a'.repeat(63)}`, true],
[`a/${'a'.repeat(64)}`, false],
])(`isValidAnnotationKey %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidAnnotationKey(value)).toBe(
matches,
);
});
it.each([
[7, false],
[null, false],
['', true],
['a', true],
['/'.repeat(6000), true],
])(`isValidAnnotationValue %p ? %p`, (value, matches) => {
expect(KubernetesValidatorFunctions.isValidAnnotationValue(value)).toBe(
matches,
);
});
});
plugins/catalog-backend/src/validation/KubernetesValidatorFunctions.ts
-86
View File
@@ -1,86 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
/**
* Contains validation functions that match the Kubernetes spec, usable to
* build a catalog that is compatible with those rule sets.
*
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
* @see https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
*/
export class KubernetesValidatorFunctions {
static isValidApiVersion(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
n => n.length >= 1 && n.length <= 63 && /^[a-z0-9A-Z]+$/.test(n),
);
}
static isValidKind(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-zA-Z][a-z0-9A-Z]*$/.test(value)
);
}
static isValidObjectName(value: any): boolean {
return (
typeof value === 'string' &&
value.length >= 1 &&
value.length <= 63 &&
/^[a-z0-9A-Z]+([-_.][a-z0-9A-Z]+)*$/.test(value)
);
}
static isValidNamespace(value: any): boolean {
return CommonValidatorFunctions.isValidDnsLabel(value);
}
static isValidLabelKey(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
KubernetesValidatorFunctions.isValidObjectName,
);
}
static isValidLabelValue(value: any): boolean {
return (
value === '' || KubernetesValidatorFunctions.isValidObjectName(value)
);
}
static isValidAnnotationKey(value: any): boolean {
return CommonValidatorFunctions.isValidPrefixAndOrSuffix(
value,
'/',
CommonValidatorFunctions.isValidDnsSubdomain,
KubernetesValidatorFunctions.isValidObjectName,
);
}
static isValidAnnotationValue(value: any): boolean {
return typeof value === 'string';
}
}
plugins/catalog-backend/src/validation/makeValidator.ts
-38
View File
@@ -1,38 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CommonValidatorFunctions } from './CommonValidatorFunctions';
import { KubernetesValidatorFunctions } from './KubernetesValidatorFunctions';
import { Validators } from './types';
const defaultValidators: Validators = {
isValidApiVersion: KubernetesValidatorFunctions.isValidApiVersion,
isValidKind: KubernetesValidatorFunctions.isValidKind,
isValidEntityName: KubernetesValidatorFunctions.isValidObjectName,
isValidNamespace: KubernetesValidatorFunctions.isValidNamespace,
normalizeEntityName: CommonValidatorFunctions.normalizeToLowercaseAlphanum,
isValidLabelKey: KubernetesValidatorFunctions.isValidLabelKey,
isValidLabelValue: KubernetesValidatorFunctions.isValidLabelValue,
isValidAnnotationKey: KubernetesValidatorFunctions.isValidAnnotationKey,
isValidAnnotationValue: KubernetesValidatorFunctions.isValidAnnotationValue,
};
export function makeValidator(overrides: Partial<Validators> = {}): Validators {
return {
...defaultValidators,
...overrides,
};
}