paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

feat(catalog-backend): make a builder to help wire up the catalog

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-10-12 15:38:12 +02:00
parent e4d7390967
commit 99710b102d
17 changed files with 925 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/2874.md
+12
View File
@@ -0,0 +1,12 @@
---
'@backstage/plugin-catalog-backend': minor
---
The way that wiring together a catalog happens, has changed drastically. Now
there is a new class `CatalogBuilder` that does almost all of the heavy lifting
of how to augment/replace pieces of catalog functionality, such as adding
support for custom entities or adding additional processors.
As the builder was added, a lot of the static methods and builders for default
setups have been removed from classes deep in the hierarchy. Instead, the
builder contains the knowledge of what the defaults are.
packages/backend-common/src/database/postgres.ts
+1 -1
View File
@@ -57,7 +57,7 @@ export function buildPgDatabaseConfig(
* Gets the postgres connection config
*
* @param dbConfig The database config
* @param parseConnectionString Flag to explictly control connection string parsing
* @param parseConnectionString Flag to explicitly control connection string parsing
*/
export function getPgConnectionConfig(
dbConfig: Config,
packages/backend/src/plugins/catalog.ts
+8 -24
View File
@@ -14,37 +14,21 @@
* limitations under the License.
*/
import { useHotCleanup } from '@backstage/backend-common';
import {
CatalogBuilder,
createRouter,
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
DatabaseManager,
HigherOrderOperations,
LocationReaders,
runPeriodically,
} from '@backstage/plugin-catalog-backend';
import { PluginEnvironment } from '../types';
import { useHotCleanup } from '@backstage/backend-common';
export default async function createPlugin({
logger,
config,
reader,
database,
}: PluginEnvironment) {
const locationReader = new LocationReaders({ logger, reader, config });
const db = await DatabaseManager.createDatabase(await database.getClient(), {
logger,
});
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const higherOrderOperation = new HigherOrderOperations(
export default async function createPlugin(env: PluginEnvironment) {
const builder = new CatalogBuilder(env);
const {
entitiesCatalog,
locationsCatalog,
locationReader,
logger,
);
higherOrderOperation,
} = await builder.build();
useHotCleanup(
module,
@@ -55,6 +39,6 @@ export default async function createPlugin({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
logger,
logger: env.logger,
});
}
packages/catalog-model/src/EntityPolicies.test.ts
+110
View File
@@ -0,0 +1,110 @@
import { Entity } from './entity';
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicies } from './EntityPolicies';
import { EntityPolicy } from './types';
describe('EntityPolicies', () => {
const p1: jest.Mocked<EntityPolicy> = { enforce: jest.fn() };
const p2: jest.Mocked<EntityPolicy> = { enforce: jest.fn() };
const entity1: Entity = {
apiVersion: 'a1',
kind: 'k1',
metadata: { name: 'n1' },
};
const entity2: Entity = {
apiVersion: 'a2',
kind: 'k2',
metadata: { name: 'n2' },
};
afterEach(() => jest.resetAllMocks());
describe('allOf', () => {
it('resolves when no policies', async () => {
const policy = EntityPolicies.allOf([]);
await expect(policy.enforce(entity1)).resolves.toBe(entity1);
});
it('resolves when all resolve', async () => {
p1.enforce.mockResolvedValue(entity1);
p2.enforce.mockResolvedValue(entity1);
const policy = EntityPolicies.allOf([p1, p2]);
await expect(policy.enforce(entity1)).resolves.toBe(entity1);
});
it('rejects when any rejects', async () => {
p1.enforce.mockResolvedValue(entity1);
p2.enforce.mockRejectedValue(new Error('a'));
const policy = EntityPolicies.allOf([p1, p2]);
await expect(policy.enforce(entity1)).rejects.toThrow('a');
});
it('rejects when any ignores', async () => {
p1.enforce.mockResolvedValue(entity1);
p2.enforce.mockResolvedValue(undefined);
const policy = EntityPolicies.allOf([p1, p2]);
await expect(policy.enforce(entity1)).rejects.toThrow(
/did not return a result/,
);
});
it('passes through transforms properly', async () => {
p1.enforce.mockResolvedValue(entity2);
p2.enforce.mockResolvedValue(entity2);
const policy = EntityPolicies.allOf([p1, p2]);
await expect(policy.enforce(entity1)).resolves.toBe(entity2);
expect(p1.enforce).toBeCalledWith(entity1);
expect(p2.enforce).toBeCalledWith(entity2);
});
});
describe('oneOf', () => {
it('rejects when no policies', async () => {
const policy = EntityPolicies.oneOf([]);
await expect(policy.enforce(entity1)).rejects.toThrow(/did not match/);
});
it('resolves when one resolves', async () => {
p1.enforce.mockResolvedValue(undefined);
p2.enforce.mockResolvedValue(entity1);
const policy = EntityPolicies.oneOf([p1, p2]);
await expect(policy.enforce(entity1)).resolves.toBe(entity1);
});
it('rejects when one rejects first', async () => {
p1.enforce.mockRejectedValue(new Error('a'));
p2.enforce.mockResolvedValue(entity1);
const policy = EntityPolicies.oneOf([p1, p2]);
await expect(policy.enforce(entity1)).rejects.toThrow('a');
});
it('resolves first resolution when several resolve', async () => {
p1.enforce.mockResolvedValue(entity1);
p2.enforce.mockResolvedValue(entity2);
const policy = EntityPolicies.oneOf([p1, p2]);
await expect(policy.enforce(entity1)).resolves.toBe(entity1);
});
it('rejects when all ignore', async () => {
p1.enforce.mockResolvedValue(undefined);
p2.enforce.mockResolvedValue(undefined);
const policy = EntityPolicies.oneOf([p1, p2]);
await expect(policy.enforce(entity1)).rejects.toThrow(/did not match/);
});
});
});
packages/catalog-model/src/EntityPolicies.ts
+7 -53
View File
@@ -14,22 +14,7 @@
* limitations under the License.
*/
import {
DefaultNamespaceEntityPolicy,
Entity,
FieldFormatEntityPolicy,
NoForeignRootFieldsEntityPolicy,
ReservedFieldsEntityPolicy,
SchemaValidEntityPolicy,
} from './entity';
import {
apiEntityV1alpha1Policy,
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
userEntityV1alpha1Policy,
} from './kinds';
import { Entity } from './entity';
import { EntityPolicy } from './types';
// Helper that requires that all of a set of policies can be successfully
@@ -68,42 +53,11 @@ class AnyEntityPolicy implements EntityPolicy {
}
}
export class EntityPolicies implements EntityPolicy {
private readonly policy: EntityPolicy;
static defaultPolicies(): EntityPolicy {
return EntityPolicies.allOf([
EntityPolicies.allOf([
new SchemaValidEntityPolicy(),
new DefaultNamespaceEntityPolicy(),
new NoForeignRootFieldsEntityPolicy(),
new FieldFormatEntityPolicy(),
new ReservedFieldsEntityPolicy(),
]),
EntityPolicies.anyOf([
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
userEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
apiEntityV1alpha1Policy,
]),
]);
}
static allOf(policies: EntityPolicy[]): EntityPolicy {
export const EntityPolicies = {
allOf(policies: EntityPolicy[]) {
return new AllEntityPolicies(policies);
}
static anyOf(policies: EntityPolicy[]): EntityPolicy {
},
oneOf(policies: EntityPolicy[]) {
return new AnyEntityPolicy(policies);
}
constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) {
this.policy = policy;
}
enforce(entity: Entity): Promise<Entity | undefined> {
return this.policy.enforce(entity);
}
}
},
};
packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts
+9 -25
View File
@@ -1,44 +1,28 @@
import { useHotCleanup } from '@backstage/backend-common';
import {
CatalogBuilder,
createRouter,
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
DatabaseManager,
HigherOrderOperations,
LocationReaders,
runPeriodically,
} from '@backstage/plugin-catalog-backend';
import { PluginEnvironment } from '../types';
import { useHotCleanup } from '@backstage/backend-common';
export default async function createPlugin({
logger,
config,
reader,
database,
}: PluginEnvironment) {
const locationReader = new LocationReaders({ logger, reader, config });
const db = await DatabaseManager.createDatabase(await database.getClient(),
{ logger },
);
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const higherOrderOperation = new HigherOrderOperations(
export default async function createPlugin(env: PluginEnvironment) {
const builder = new CatalogBuilder(env);
const {
entitiesCatalog,
locationsCatalog,
locationReader,
logger,
);
higherOrderOperation,
} = await builder.build();
useHotCleanup(
module,
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000),
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
);
return await createRouter({
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
logger,
logger: env.logger,
});
}
plugins/catalog-backend/src/database/DatabaseManager.ts
+15 -9
View File
@@ -46,21 +46,31 @@ export class DatabaseManager {
return new CommonDatabase(knex, logger);
}
public static async createInMemoryDatabase(
options: Partial<CreateDatabaseOptions> = {},
): Promise<Database> {
public static async createInMemoryDatabase(): Promise<Database> {
const knex = await this.createInMemoryDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createInMemoryDatabaseConnection(): Promise<Knex> {
const knex = Knex({
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
});
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return DatabaseManager.createDatabase(knex, options);
return knex;
}
public static async createTestDatabase(): Promise<Database> {
const knex = await this.createTestDatabaseConnection();
return await this.createDatabase(knex);
}
public static async createTestDatabaseConnection(): Promise<Knex> {
const config: Knex.Config<any> = {
/*
client: 'pg',
@@ -91,11 +101,7 @@ export class DatabaseManager {
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
await knex.migrate.latest({
directory: migrationsDir,
});
const { logger } = defaultOptions;
return new CommonDatabase(knex, logger);
return knex;
}
}
plugins/catalog-backend/src/index.ts
+1 -1
View File
@@ -17,5 +17,5 @@
export * from './catalog';
export * from './database';
export * from './ingestion';
export * from './service/router';
export * from './service';
export * from './util';
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+31 -110
View File
@@ -14,32 +14,16 @@
* limitations under the License.
*/
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { UrlReader } from '@backstage/backend-common';
import {
Entity,
EntityPolicies,
EntityPolicy,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
} from '@backstage/catalog-model';
import { Config, ConfigReader } from '@backstage/config';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { CatalogRulesEnforcer } from './CatalogRules';
import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor';
import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor';
import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
import { FileReaderProcessor } from './processors/FileReaderProcessor';
import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
import { LdapOrgReaderProcessor } from './processors/LdapOrgReaderProcessor';
import { LocationRefProcessor } from './processors/LocationEntityProcessor';
import { PlaceholderProcessor } from './processors/PlaceholderProcessor';
import * as result from './processors/results';
import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
import {
LocationProcessor,
LocationProcessorDataResult,
@@ -49,8 +33,6 @@ import {
LocationProcessorLocationResult,
LocationProcessorResult,
} from './processors/types';
import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
import { YamlProcessor } from './processors/YamlProcessor';
import { LocationReader, ReadLocationResult } from './types';
// The max amount of nesting depth of generated work items
@@ -58,94 +40,25 @@ const MAX_DEPTH = 10;
type Options = {
reader: UrlReader;
logger?: Logger;
config?: Config;
processors?: LocationProcessor[];
logger: Logger;
config: Config;
processors: LocationProcessor[];
rulesEnforcer: CatalogRulesEnforcer;
};
/**
* Implements the reading of a location through a series of processor tasks.
*/
export class LocationReaders implements LocationReader {
private readonly logger: Logger;
private readonly processors: LocationProcessor[];
private readonly rulesEnforcer: CatalogRulesEnforcer;
private readonly options: Options;
static defaultProcessors(options: {
logger: Logger;
reader: UrlReader;
config?: Config;
entityPolicy?: EntityPolicy;
}): LocationProcessor[] {
const {
logger,
config = new ConfigReader({}, 'missing-config'),
entityPolicy = new EntityPolicies(),
} = options;
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to the new
// integration config driven UrlReaders. In an upcoming release we can then completely
// remove support for the old processors, but still keep handling the deprecated location
// types for a while, but with a warning.
const oldProcessors = [];
const pc = config.getOptionalConfig('catalog.processors');
if (pc?.has('github')) {
logger.warn(
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
);
oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger));
}
if (pc?.has('gitlabApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
);
oldProcessors.push(new GitlabApiReaderProcessor(config));
oldProcessors.push(new GitlabReaderProcessor());
}
if (pc?.has('bitbucketApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
);
oldProcessors.push(new BitbucketApiReaderProcessor(config));
}
if (pc?.has('azureApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
);
oldProcessors.push(new AzureApiReaderProcessor(config));
}
return [
StaticLocationProcessor.fromConfig(config),
new FileReaderProcessor(),
...oldProcessors,
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor(options),
new YamlProcessor(),
PlaceholderProcessor.default({ reader: options.reader }),
new CodeOwnersProcessor({ reader: options.reader }),
new EntityPolicyProcessor(entityPolicy),
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
];
}
constructor({
logger = getVoidLogger(),
config,
reader,
processors = LocationReaders.defaultProcessors({ logger, reader, config }),
}: Options) {
this.logger = logger;
this.processors = processors;
this.rulesEnforcer = config
? CatalogRulesEnforcer.fromConfig(config)
: new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules);
constructor(options: Options) {
this.options = options;
}
async read(location: LocationSpec): Promise<ReadLocationResult> {
const { rulesEnforcer, logger } = this.options;
const output: ReadLocationResult = { entities: [], errors: [] };
let items: LocationProcessorResult[] = [result.location(location, false)];
@@ -159,7 +72,7 @@ export class LocationReaders implements LocationReader {
} else if (item.type === 'data') {
await this.handleData(item, emit);
} else if (item.type === 'entity') {
if (this.rulesEnforcer.isAllowed(item.entity, item.location)) {
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
const entity = await this.handleEntity(item, emit);
output.entities.push({
entity,
@@ -190,7 +103,7 @@ export class LocationReaders implements LocationReader {
}
const message = `Max recursion depth ${MAX_DEPTH} reached for ${location.type} ${location.target}`;
this.logger.warn(message);
logger.warn(message);
output.errors.push({ location, error: new Error(message) });
return output;
}
@@ -199,7 +112,9 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorLocationResult,
emit: LocationProcessorEmit,
) {
for (const processor of this.processors) {
const { processors, logger } = this.options;
for (const processor of processors) {
if (processor.readLocation) {
try {
if (
@@ -210,21 +125,23 @@ export class LocationReaders implements LocationReader {
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
logger.warn(message);
}
}
}
const message = `No processor was able to read location ${item.location.type} ${item.location.target}`;
emit(result.inputError(item.location, message));
this.logger.warn(message);
logger.warn(message);
}
private async handleData(
item: LocationProcessorDataResult,
emit: LocationProcessorEmit,
) {
for (const processor of this.processors) {
const { processors, logger } = this.options;
for (const processor of processors) {
if (processor.parseData) {
try {
if (await processor.parseData(item.data, item.location, emit)) {
@@ -233,7 +150,7 @@ export class LocationReaders implements LocationReader {
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
logger.warn(message);
}
}
}
@@ -246,9 +163,11 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorEntityResult,
emit: LocationProcessorEmit,
): Promise<Entity> {
const { processors, logger } = this.options;
let current = item.entity;
for (const processor of this.processors) {
for (const processor of processors) {
if (processor.processEntity) {
try {
current = await processor.processEntity(current, item.location, emit);
@@ -261,7 +180,7 @@ export class LocationReaders implements LocationReader {
const name = !current.metadata ? '' : current.metadata.name;
const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
logger.warn(message);
}
}
}
@@ -273,18 +192,20 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorErrorResult,
emit: LocationProcessorEmit,
) {
this.logger.debug(
const { processors, logger } = this.options;
logger.debug(
`Encountered error at location ${item.location.type} ${item.location.target}, ${item.error}`,
);
for (const processor of this.processors) {
for (const processor of processors) {
if (processor.handleError) {
try {
await processor.handleError(item.error, item.location, emit);
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
logger.warn(message);
}
}
}
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts
+29 -10
View File
@@ -21,6 +21,7 @@ import {
PlaceholderResolver,
ResolverParams,
ResolverRead,
textPlaceholderResolver,
yamlPlaceholderResolver,
} from './PlaceholderProcessor';
@@ -134,9 +135,12 @@ describe('PlaceholderProcessor', () => {
expect(read).not.toBeCalled();
});
it('has builtin text support', async () => {
it('works with the text resolver', async () => {
read.mockResolvedValue(Buffer.from('TEXT', 'utf-8'));
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { text: textPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
@@ -163,11 +167,14 @@ describe('PlaceholderProcessor', () => {
);
});
it('has builtin json support', async () => {
it('works with the json resolver', async () => {
read.mockResolvedValue(
Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'),
);
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { json: jsonPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
@@ -194,9 +201,12 @@ describe('PlaceholderProcessor', () => {
);
});
it('has builtin yaml support', async () => {
it('works with the yaml resolver', async () => {
read.mockResolvedValue(Buffer.from('foo:\n - bar: 7', 'utf-8'));
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { yaml: yamlPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
@@ -225,7 +235,10 @@ describe('PlaceholderProcessor', () => {
it('resolves absolute path for absolute location', async () => {
read.mockResolvedValue(Buffer.from('TEXT', 'utf-8'));
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { text: textPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
@@ -258,7 +271,10 @@ describe('PlaceholderProcessor', () => {
it('resolves absolute path for relative file location', async () => {
read.mockResolvedValue(Buffer.from('TEXT', 'utf-8'));
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { text: textPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
@@ -291,10 +307,13 @@ describe('PlaceholderProcessor', () => {
it('not resolves relative file path for relative file location', async () => {
// We explicitly don't support this case, as it would allow for file system
// traversel attacks. If we want to implement this, we need to have additional
// traversal attacks. If we want to implement this, we need to have additional
// security measures in place!
read.mockResolvedValue(Buffer.from('TEXT', 'utf-8'));
const processor = PlaceholderProcessor.default({ reader });
const processor = new PlaceholderProcessor({
resolvers: { text: textPlaceholderResolver },
reader,
});
await expect(
processor.processEntity(
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts
-11
View File
@@ -43,17 +43,6 @@ type Options = {
* that it then fills in with actual data.
*/
export class PlaceholderProcessor implements LocationProcessor {
static default({ reader }: { reader: UrlReader }) {
return new PlaceholderProcessor({
resolvers: {
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
},
reader,
});
}
constructor(private readonly options: Options) {}
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts
+5
View File
@@ -20,6 +20,11 @@ import yaml from 'yaml';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
/**
* Handles incoming raw data buffers, and if they have a yaml extension,
* attempts to parse them into structured data and emitting them as un-
* validated entities.
*/
export class YamlProcessor implements LocationProcessor {
async parseData(
data: Buffer,
plugins/catalog-backend/src/ingestion/processors/index.ts
+1
View File
@@ -31,6 +31,7 @@ export { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor';
export { GitlabReaderProcessor } from './GitlabReaderProcessor';
export { LocationRefProcessor } from './LocationEntityProcessor';
export { PlaceholderProcessor } from './PlaceholderProcessor';
export type { PlaceholderResolver } from './PlaceholderProcessor';
export { StaticLocationProcessor } from './StaticLocationProcessor';
export { UrlReaderProcessor } from './UrlReaderProcessor';
export { YamlProcessor } from './YamlProcessor';
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+139
View File
@@ -0,0 +1,139 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { Entity, LocationSpec } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { DatabaseManager } from '../database';
import { LocationProcessorEmit } from '../ingestion';
import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder';
import * as result from '../ingestion/processors/results';
describe('CatalogBuilder', () => {
const db = DatabaseManager.createTestDatabaseConnection();
const reader: jest.Mocked<UrlReader> = { read: jest.fn() };
const env: CatalogEnvironment = {
logger: getVoidLogger(),
database: { getClient: () => db },
config: ConfigReader.fromConfigs([]),
reader,
};
afterEach(() => jest.resetAllMocks());
it('works with no changes', async () => {
const builder = new CatalogBuilder(env);
const built = await builder.build();
await expect(built.entitiesCatalog.entities()).resolves.toEqual([]);
await expect(built.locationsCatalog.locations()).resolves.toEqual([
expect.objectContaining({
data: expect.objectContaining({ type: 'bootstrap' }),
}),
]);
});
it('works with everything replaced', async () => {
reader.read.mockResolvedValue(Buffer.from('junk'));
const builder = new CatalogBuilder(env)
.replaceReaderProcessors([
{
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: LocationProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
},
])
.replaceParserProcessors([
{
async parseData(
data: Buffer,
location: LocationSpec,
emit: LocationProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
kind: 'Component',
metadata: { name: 'n' },
}),
);
return true;
},
},
])
.replacePreProcessors([
{
async processEntity(entity: Entity) {
expect(entity.apiVersion).toBe('av');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
},
])
.replaceEntityPolicies([
{
async enforce(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return entity;
},
},
])
.replaceEntityKinds([
{
async enforce(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return entity;
},
},
])
.replacePostProcessors([
{
async processEntity(entity: Entity) {
return {
...entity,
metadata: { ...entity.metadata, post: 'p' },
};
},
},
]);
const out = await builder.build();
const added = await out.higherOrderOperation.addLocation({
type: 'test',
target: '',
});
expect(added.entities).toEqual([
{
apiVersion: 'av',
kind: 'Component',
metadata: expect.objectContaining({
name: 'n',
namespace: 'ns',
post: 'p',
}),
},
]);
});
});
plugins/catalog-backend/src/service/CatalogBuilder.ts
+523
View File
@@ -0,0 +1,523 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common';
import {
apiEntityV1alpha1Policy,
componentEntityV1alpha1Policy,
DefaultNamespaceEntityPolicy,
EntityPolicies,
EntityPolicy,
FieldFormatEntityPolicy,
groupEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
makeValidator,
NoForeignRootFieldsEntityPolicy,
ReservedFieldsEntityPolicy,
SchemaValidEntityPolicy,
templateEntityV1alpha1Policy,
userEntityV1alpha1Policy,
Validators,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import lodash from 'lodash';
import { Logger } from 'winston';
import {
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
EntitiesCatalog,
LocationsCatalog,
} from '../catalog';
import { DatabaseManager } from '../database';
import {
AnnotateLocationEntityProcessor,
AzureApiReaderProcessor,
BitbucketApiReaderProcessor,
CodeOwnersProcessor,
EntityPolicyProcessor,
FileReaderProcessor,
GithubOrgReaderProcessor,
GithubReaderProcessor,
GitlabApiReaderProcessor,
GitlabReaderProcessor,
HigherOrderOperation,
HigherOrderOperations,
LocationProcessor,
LocationReaders,
LocationRefProcessor,
PlaceholderProcessor,
PlaceholderResolver,
StaticLocationProcessor,
UrlReaderProcessor,
YamlProcessor,
} from '../ingestion';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor';
import {
jsonPlaceholderResolver,
textPlaceholderResolver,
yamlPlaceholderResolver,
} from '../ingestion/processors/PlaceholderProcessor';
export type CatalogEnvironment = {
logger: Logger;
database: PluginDatabaseManager;
config: Config;
reader: UrlReader;
};
/**
* A builder that helps wire up all of the component parts of the catalog.
*
* The touch points where you can replace or extend behavior are as follows:
*
* - Reader processors can be added or replaced. These implement the
* functionality of reading raw data from a location, in the form of an
* entity definition file.
* - Parser processors can be added or replaced. These accept the raw data as
* read by the previous processors and parse it into raw structured data
* (for example, from a binary buffer containing yaml text, to a JS object
* structure).
* - Placeholder resolvers can be replaced or added. These run on the raw
* structured data between the parsing and pre-processing steps, to replace
* dollar-prefixed entries with their actual values (like $file).
* - Pre-processors can be added or replaced. These take the raw unvalidated
* data from the parser processors and can enrich or extend it before
* validation. This is the place where for example codeowners data can be
* injected into partial entity definitions.
* - Entity policies can be added or replaced. These are the first line of
* validation from the output of the pre-processing step. All policies are
* given the chance to inspect the entity, and all of them have to pass in
* order for the entity to be considered valid from an overall point of
* view.
* - Field format validators can be replaced. These check the format of
* individual core fields such as metadata.name, such that they adhere to
* certain rules.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which add additional
* kind-specific validation (usually based on a schema). Only one of the
* entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Post-processors can be added or replaced. These take the validated
* entities out of the validation step and can perform additional actions on
* them.
*/
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
private entityPolicies: EntityPolicy[];
private entityPoliciesReplace: boolean;
private entityKinds: EntityPolicy[];
private entityKindsReplace: boolean;
private readerProcessors: LocationProcessor[];
private readerProcessorsReplace: boolean;
private parserProcessors: LocationProcessor[];
private parserProcessorsReplace: boolean;
private preProcessors: LocationProcessor[];
private preProcessorsReplace: boolean;
private postProcessors: LocationProcessor[];
private postProcessorsReplace: boolean;
private placeholderResolvers: Record<string, PlaceholderResolver>;
private fieldFormatValidators: Partial<Validators>;
constructor(env: CatalogEnvironment) {
this.env = env;
this.entityPolicies = [];
this.entityPoliciesReplace = false;
this.entityKinds = [];
this.entityKindsReplace = false;
this.readerProcessors = [];
this.readerProcessorsReplace = false;
this.parserProcessors = [];
this.parserProcessorsReplace = false;
this.preProcessors = [];
this.preProcessorsReplace = false;
this.postProcessors = [];
this.postProcessorsReplace = false;
this.placeholderResolvers = {};
this.fieldFormatValidators = {};
}
/**
* Adds policies that are used to validate entities between the pre-
* processing and post-processing stages. All such policies must pass for the
* entity to be considered valid.
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* @param policies One or more policies
*/
addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies.push(...policies);
return this;
}
/**
* Sets what policies to use for validation of entities between the pre-
* processing and post-processing stages. All such policies must pass for the
* entity to be considered valid.
*
* If what you want to do is to replace the rules for what format is allowed
* in various core entity fields (such as metadata.name), you may want to use
* {@link CatalogBuilder#setFieldFormatValidators} instead.
*
* This function replaces the default set of policies; use with care.
*
* @param policies One or more policies
*/
replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {
this.entityPolicies = [...policies];
this.entityPoliciesReplace = true;
return this;
}
/**
* Adds entity kinds that are used to validate a certain apiVersion/kind. One
* of the entity kind policies must match a given entity for it to be
* considered valid.
*
* @param policies One or more policies
*/
addEntityKind(...policies: EntityPolicy[]): CatalogBuilder {
this.entityKinds.push(...policies);
return this;
}
/**
* Sets what entity policies that are used to validate a certain apiVersion/
* kind. One of the entity kind policies must match a given entity for it to
* be considered valid.
*
* This function replaces the default set of kinds; use with care.
*
* @param policies One or more policies
*/
replaceEntityKinds(policies: EntityPolicy[]): CatalogBuilder {
this.entityKinds = [...policies];
this.entityKindsReplace = true;
return this;
}
/**
* Adds processors that support reading of definition files. These are run
* before the entities are parsed, pre-processed, validated and post-
* processed.
*
* @param processors One or more processors
*/
addReaderProcessor(...processors: LocationProcessor[]): CatalogBuilder {
this.readerProcessors.push(...processors);
return this;
}
/**
* Sets what processors to use for the reading of definition files. These are
* run before the entities are parsed, pre-processed, validated and post-
* processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceReaderProcessors(processors: LocationProcessor[]): CatalogBuilder {
this.readerProcessors = [...processors];
this.readerProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* @param processors One or more processors
*/
addParserProcessor(...processors: LocationProcessor[]): CatalogBuilder {
this.parserProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceParserProcessors(processors: LocationProcessor[]): CatalogBuilder {
this.parserProcessors = [...processors];
this.parserProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* @param processors One or more processors
*/
addPreProcessor(...processors: LocationProcessor[]): CatalogBuilder {
this.preProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePreProcessors(processors: LocationProcessor[]): CatalogBuilder {
this.preProcessors = [...processors];
this.preProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* @param processors One or more processors
*/
addPostProcessor(...processors: LocationProcessor[]): CatalogBuilder {
this.postProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePostProcessors(processors: LocationProcessor[]): CatalogBuilder {
this.postProcessors = [...processors];
this.postProcessorsReplace = true;
return this;
}
/**
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
* definition files.
*
* @param key The key that identifies the placeholder, e.g. "file"
* @param resolver The resolver that gets values for this placeholder
*/
setPlaceholderResolver(key: string, resolver: PlaceholderResolver) {
this.placeholderResolvers[key] = resolver;
}
/**
* Sets the validator function to use for one or more special fields of an
* entity. This is useful if the default rules for formatting of fields are
* not sufficient.
*
* This function has no effect if used together with
* {@link CatalogBuilder#replaceEntityPolicies}.
*
* @param validators The (subset of) validators to set
*/
setFieldFormatValidators(validators: Partial<Validators>) {
lodash.merge(this.fieldFormatValidators, validators);
}
/**
* Wires up and returns all of the component parts of the catalog
*/
async build(): Promise<{
entitiesCatalog: EntitiesCatalog;
locationsCatalog: LocationsCatalog;
higherOrderOperation: HigherOrderOperation;
}> {
const { config, database, logger } = this.env;
const entityPolicy = this.buildEntityPolicy();
const processors = this.buildProcessors(entityPolicy);
const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config);
const locationReader = new LocationReaders({
...this.env,
processors,
rulesEnforcer,
});
const db = await DatabaseManager.createDatabase(
await database.getClient(),
{ logger },
);
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const higherOrderOperation = new HigherOrderOperations(
entitiesCatalog,
locationsCatalog,
locationReader,
logger,
);
return {
entitiesCatalog,
locationsCatalog,
higherOrderOperation,
};
}
private buildEntityPolicy(): EntityPolicy {
const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace
? [new SchemaValidEntityPolicy(), ...this.entityPolicies]
: [
new SchemaValidEntityPolicy(),
new DefaultNamespaceEntityPolicy(),
new NoForeignRootFieldsEntityPolicy(),
new FieldFormatEntityPolicy(
makeValidator(this.fieldFormatValidators),
),
new ReservedFieldsEntityPolicy(),
...this.entityPolicies,
];
const entityKinds: EntityPolicy[] = this.entityKindsReplace
? this.entityKinds
: [
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
userEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
apiEntityV1alpha1Policy,
...this.entityKinds,
];
return EntityPolicies.allOf([
EntityPolicies.allOf(entityPolicies),
EntityPolicies.oneOf(entityKinds),
]);
}
private buildProcessors(entityPolicy: EntityPolicy): LocationProcessor[] {
const { config, reader } = this.env;
const placeholderResolvers = lodash.merge(
{
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
},
this.placeholderResolvers,
);
return [
StaticLocationProcessor.fromConfig(config),
...this.buildReaderProcessors(),
...this.buildParserProcessors(),
new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }),
...this.buildPreProcessors(),
new EntityPolicyProcessor(entityPolicy),
...this.buildPostProcessors(),
];
}
private buildReaderProcessors(): LocationProcessor[] {
const { config, logger, reader } = this.env;
if (this.readerProcessorsReplace) {
return this.readerProcessors;
}
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to the new
// integration config driven UrlReaders. In an upcoming release we can then completely
// remove support for the old processors, but still keep handling the deprecated location
// types for a while, but with a warning.
const oldProcessors = [];
const pc = config.getOptionalConfig('catalog.processors');
if (pc?.has('github')) {
logger.warn(
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
);
oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger));
}
if (pc?.has('gitlabApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
);
oldProcessors.push(new GitlabApiReaderProcessor(config));
oldProcessors.push(new GitlabReaderProcessor());
}
if (pc?.has('bitbucketApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
);
oldProcessors.push(new BitbucketApiReaderProcessor(config));
}
if (pc?.has('azureApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
);
oldProcessors.push(new AzureApiReaderProcessor(config));
}
return [
new FileReaderProcessor(),
...oldProcessors,
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
...this.readerProcessors,
];
}
private buildParserProcessors(): LocationProcessor[] {
if (this.parserProcessorsReplace) {
return this.parserProcessors;
}
return [new YamlProcessor(), ...this.parserProcessors];
}
private buildPreProcessors(): LocationProcessor[] {
const { reader } = this.env;
if (this.preProcessorsReplace) {
return this.preProcessors;
}
return [new CodeOwnersProcessor({ reader }), ...this.preProcessors];
}
private buildPostProcessors(): LocationProcessor[] {
if (this.postProcessorsReplace) {
return this.postProcessors;
}
return [
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
...this.postProcessors,
];
}
}
plugins/catalog-backend/src/service/index.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { CatalogBuilder } from './CatalogBuilder';
export { createRouter } from './router';
export type { RouterOptions } from './router';
plugins/catalog-backend/src/service/standaloneServer.ts
+15 -13
View File
@@ -18,16 +18,14 @@ import {
createServiceBuilder,
loadBackendConfig,
UrlReaders,
useHotMemoize,
} from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { Server } from 'http';
import { Logger } from 'winston';
import { HigherOrderOperations } from '..';
import { DatabaseEntitiesCatalog } from '../catalog/DatabaseEntitiesCatalog';
import { DatabaseLocationsCatalog } from '../catalog/DatabaseLocationsCatalog';
import { DatabaseManager } from '../database/DatabaseManager';
import { DatabaseManager } from '../database';
import { CatalogBuilder } from './CatalogBuilder';
import { createRouter } from './router';
import { LocationReaders } from '../ingestion';
export interface ServerOptions {
port: number;
@@ -41,18 +39,22 @@ export async function startStandaloneServer(
const logger = options.logger.child({ service: 'catalog-backend' });
const config = ConfigReader.fromConfigs(await loadBackendConfig());
const reader = UrlReaders.default({ logger, config });
const db = useHotMemoize(module, () =>
DatabaseManager.createInMemoryDatabaseConnection(),
);
logger.debug('Creating application...');
const db = await DatabaseManager.createInMemoryDatabase({ logger });
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
const locationsCatalog = new DatabaseLocationsCatalog(db);
const locationReader = new LocationReaders({ logger, config, reader });
const higherOrderOperation = new HigherOrderOperations(
const builder = new CatalogBuilder({
logger,
database: { getClient: () => db },
config,
reader,
});
const {
entitiesCatalog,
locationsCatalog,
locationReader,
logger,
);
higherOrderOperation,
} = await builder.build();
logger.debug('Starting application server...');
const router = await createRouter({