From 99710b102de57b688bb94e46b4d3e39f333e5a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 12 Oct 2020 15:38:12 +0200 Subject: [PATCH] feat(catalog-backend): make a builder to help wire up the catalog --- .changeset/2874.md | 12 + .../backend-common/src/database/postgres.ts | 2 +- packages/backend/src/plugins/catalog.ts | 32 +- .../catalog-model/src/EntityPolicies.test.ts | 110 ++++ packages/catalog-model/src/EntityPolicies.ts | 60 +- .../packages/backend/src/plugins/catalog.ts | 34 +- .../src/database/DatabaseManager.ts | 24 +- plugins/catalog-backend/src/index.ts | 2 +- .../src/ingestion/LocationReaders.ts | 141 ++--- .../processors/PlaceholderProcessor.test.ts | 39 +- .../processors/PlaceholderProcessor.ts | 11 - .../src/ingestion/processors/YamlProcessor.ts | 5 + .../src/ingestion/processors/index.ts | 1 + .../src/service/CatalogBuilder.test.ts | 139 +++++ .../src/service/CatalogBuilder.ts | 523 ++++++++++++++++++ plugins/catalog-backend/src/service/index.ts | 19 + .../src/service/standaloneServer.ts | 28 +- 17 files changed, 925 insertions(+), 257 deletions(-) create mode 100644 .changeset/2874.md create mode 100644 packages/catalog-model/src/EntityPolicies.test.ts create mode 100644 plugins/catalog-backend/src/service/CatalogBuilder.test.ts create mode 100644 plugins/catalog-backend/src/service/CatalogBuilder.ts create mode 100644 plugins/catalog-backend/src/service/index.ts diff --git a/.changeset/2874.md b/.changeset/2874.md new file mode 100644 index 0000000000..693962314f --- /dev/null +++ b/.changeset/2874.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +The way that wiring together a catalog happens, has changed drastically. Now +there is a new class `CatalogBuilder` that does almost all of the heavy lifting +of how to augment/replace pieces of catalog functionality, such as adding +support for custom entities or adding additional processors. + +As the builder was added, a lot of the static methods and builders for default +setups have been removed from classes deep in the hierarchy. Instead, the +builder contains the knowledge of what the defaults are. diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 82e971f081..04bd3615b6 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -57,7 +57,7 @@ export function buildPgDatabaseConfig( * Gets the postgres connection config * * @param dbConfig The database config - * @param parseConnectionString Flag to explictly control connection string parsing + * @param parseConnectionString Flag to explicitly control connection string parsing */ export function getPgConnectionConfig( dbConfig: Config, diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index b8e99e6398..3c205a3800 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -14,37 +14,21 @@ * limitations under the License. */ +import { useHotCleanup } from '@backstage/backend-common'; import { + CatalogBuilder, createRouter, - DatabaseEntitiesCatalog, - DatabaseLocationsCatalog, - DatabaseManager, - HigherOrderOperations, - LocationReaders, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; -import { useHotCleanup } from '@backstage/backend-common'; -export default async function createPlugin({ - logger, - config, - reader, - database, -}: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, reader, config }); - - const db = await DatabaseManager.createDatabase(await database.getClient(), { - logger, - }); - const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db); - const higherOrderOperation = new HigherOrderOperations( +export default async function createPlugin(env: PluginEnvironment) { + const builder = new CatalogBuilder(env); + const { entitiesCatalog, locationsCatalog, - locationReader, - logger, - ); + higherOrderOperation, + } = await builder.build(); useHotCleanup( module, @@ -55,6 +39,6 @@ export default async function createPlugin({ entitiesCatalog, locationsCatalog, higherOrderOperation, - logger, + logger: env.logger, }); } diff --git a/packages/catalog-model/src/EntityPolicies.test.ts b/packages/catalog-model/src/EntityPolicies.test.ts new file mode 100644 index 0000000000..cd869f798c --- /dev/null +++ b/packages/catalog-model/src/EntityPolicies.test.ts @@ -0,0 +1,110 @@ +import { Entity } from './entity'; +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicies } from './EntityPolicies'; +import { EntityPolicy } from './types'; + +describe('EntityPolicies', () => { + const p1: jest.Mocked = { enforce: jest.fn() }; + const p2: jest.Mocked = { enforce: jest.fn() }; + const entity1: Entity = { + apiVersion: 'a1', + kind: 'k1', + metadata: { name: 'n1' }, + }; + const entity2: Entity = { + apiVersion: 'a2', + kind: 'k2', + metadata: { name: 'n2' }, + }; + + afterEach(() => jest.resetAllMocks()); + + describe('allOf', () => { + it('resolves when no policies', async () => { + const policy = EntityPolicies.allOf([]); + await expect(policy.enforce(entity1)).resolves.toBe(entity1); + }); + + it('resolves when all resolve', async () => { + p1.enforce.mockResolvedValue(entity1); + p2.enforce.mockResolvedValue(entity1); + const policy = EntityPolicies.allOf([p1, p2]); + await expect(policy.enforce(entity1)).resolves.toBe(entity1); + }); + + it('rejects when any rejects', async () => { + p1.enforce.mockResolvedValue(entity1); + p2.enforce.mockRejectedValue(new Error('a')); + const policy = EntityPolicies.allOf([p1, p2]); + await expect(policy.enforce(entity1)).rejects.toThrow('a'); + }); + + it('rejects when any ignores', async () => { + p1.enforce.mockResolvedValue(entity1); + p2.enforce.mockResolvedValue(undefined); + const policy = EntityPolicies.allOf([p1, p2]); + await expect(policy.enforce(entity1)).rejects.toThrow( + /did not return a result/, + ); + }); + + it('passes through transforms properly', async () => { + p1.enforce.mockResolvedValue(entity2); + p2.enforce.mockResolvedValue(entity2); + const policy = EntityPolicies.allOf([p1, p2]); + await expect(policy.enforce(entity1)).resolves.toBe(entity2); + expect(p1.enforce).toBeCalledWith(entity1); + expect(p2.enforce).toBeCalledWith(entity2); + }); + }); + + describe('oneOf', () => { + it('rejects when no policies', async () => { + const policy = EntityPolicies.oneOf([]); + await expect(policy.enforce(entity1)).rejects.toThrow(/did not match/); + }); + + it('resolves when one resolves', async () => { + p1.enforce.mockResolvedValue(undefined); + p2.enforce.mockResolvedValue(entity1); + const policy = EntityPolicies.oneOf([p1, p2]); + await expect(policy.enforce(entity1)).resolves.toBe(entity1); + }); + + it('rejects when one rejects first', async () => { + p1.enforce.mockRejectedValue(new Error('a')); + p2.enforce.mockResolvedValue(entity1); + const policy = EntityPolicies.oneOf([p1, p2]); + await expect(policy.enforce(entity1)).rejects.toThrow('a'); + }); + + it('resolves first resolution when several resolve', async () => { + p1.enforce.mockResolvedValue(entity1); + p2.enforce.mockResolvedValue(entity2); + const policy = EntityPolicies.oneOf([p1, p2]); + await expect(policy.enforce(entity1)).resolves.toBe(entity1); + }); + + it('rejects when all ignore', async () => { + p1.enforce.mockResolvedValue(undefined); + p2.enforce.mockResolvedValue(undefined); + const policy = EntityPolicies.oneOf([p1, p2]); + await expect(policy.enforce(entity1)).rejects.toThrow(/did not match/); + }); + }); +}); diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index acdfb4df2a..1f304f8f27 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -14,22 +14,7 @@ * limitations under the License. */ -import { - DefaultNamespaceEntityPolicy, - Entity, - FieldFormatEntityPolicy, - NoForeignRootFieldsEntityPolicy, - ReservedFieldsEntityPolicy, - SchemaValidEntityPolicy, -} from './entity'; -import { - apiEntityV1alpha1Policy, - componentEntityV1alpha1Policy, - groupEntityV1alpha1Policy, - locationEntityV1alpha1Policy, - templateEntityV1alpha1Policy, - userEntityV1alpha1Policy, -} from './kinds'; +import { Entity } from './entity'; import { EntityPolicy } from './types'; // Helper that requires that all of a set of policies can be successfully @@ -68,42 +53,11 @@ class AnyEntityPolicy implements EntityPolicy { } } -export class EntityPolicies implements EntityPolicy { - private readonly policy: EntityPolicy; - - static defaultPolicies(): EntityPolicy { - return EntityPolicies.allOf([ - EntityPolicies.allOf([ - new SchemaValidEntityPolicy(), - new DefaultNamespaceEntityPolicy(), - new NoForeignRootFieldsEntityPolicy(), - new FieldFormatEntityPolicy(), - new ReservedFieldsEntityPolicy(), - ]), - EntityPolicies.anyOf([ - componentEntityV1alpha1Policy, - groupEntityV1alpha1Policy, - userEntityV1alpha1Policy, - locationEntityV1alpha1Policy, - templateEntityV1alpha1Policy, - apiEntityV1alpha1Policy, - ]), - ]); - } - - static allOf(policies: EntityPolicy[]): EntityPolicy { +export const EntityPolicies = { + allOf(policies: EntityPolicy[]) { return new AllEntityPolicies(policies); - } - - static anyOf(policies: EntityPolicy[]): EntityPolicy { + }, + oneOf(policies: EntityPolicy[]) { return new AnyEntityPolicy(policies); - } - - constructor(policy: EntityPolicy = EntityPolicies.defaultPolicies()) { - this.policy = policy; - } - - enforce(entity: Entity): Promise { - return this.policy.enforce(entity); - } -} + }, +}; diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts index aee38a5fa0..10ac116ded 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -1,44 +1,28 @@ +import { useHotCleanup } from '@backstage/backend-common'; import { + CatalogBuilder, createRouter, - DatabaseEntitiesCatalog, - DatabaseLocationsCatalog, - DatabaseManager, - HigherOrderOperations, - LocationReaders, runPeriodically, } from '@backstage/plugin-catalog-backend'; import { PluginEnvironment } from '../types'; -import { useHotCleanup } from '@backstage/backend-common'; -export default async function createPlugin({ - logger, - config, - reader, - database, -}: PluginEnvironment) { - const locationReader = new LocationReaders({ logger, reader, config }); - - const db = await DatabaseManager.createDatabase(await database.getClient(), - { logger }, - ); - const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db); - const higherOrderOperation = new HigherOrderOperations( +export default async function createPlugin(env: PluginEnvironment) { + const builder = new CatalogBuilder(env); + const { entitiesCatalog, locationsCatalog, - locationReader, - logger, - ); + higherOrderOperation, + } = await builder.build(); useHotCleanup( module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000), + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), ); return await createRouter({ entitiesCatalog, locationsCatalog, higherOrderOperation, - logger, + logger: env.logger, }); } diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts index eb307c9478..465bf654f2 100644 --- a/plugins/catalog-backend/src/database/DatabaseManager.ts +++ b/plugins/catalog-backend/src/database/DatabaseManager.ts @@ -46,21 +46,31 @@ export class DatabaseManager { return new CommonDatabase(knex, logger); } - public static async createInMemoryDatabase( - options: Partial = {}, - ): Promise { + public static async createInMemoryDatabase(): Promise { + const knex = await this.createInMemoryDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createInMemoryDatabaseConnection(): Promise { const knex = Knex({ client: 'sqlite3', connection: ':memory:', useNullAsDefault: true, }); + knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); - return DatabaseManager.createDatabase(knex, options); + + return knex; } public static async createTestDatabase(): Promise { + const knex = await this.createTestDatabaseConnection(); + return await this.createDatabase(knex); + } + + public static async createTestDatabaseConnection(): Promise { const config: Knex.Config = { /* client: 'pg', @@ -91,11 +101,7 @@ export class DatabaseManager { knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); }); - await knex.migrate.latest({ - directory: migrationsDir, - }); - const { logger } = defaultOptions; - return new CommonDatabase(knex, logger); + return knex; } } diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index 89fd4d37ed..f621930d10 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -17,5 +17,5 @@ export * from './catalog'; export * from './database'; export * from './ingestion'; -export * from './service/router'; +export * from './service'; export * from './util'; diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 87c75e21e5..0f6ad18f97 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -14,32 +14,16 @@ * limitations under the License. */ -import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { UrlReader } from '@backstage/backend-common'; import { Entity, - EntityPolicies, - EntityPolicy, ENTITY_DEFAULT_NAMESPACE, LocationSpec, } from '@backstage/catalog-model'; -import { Config, ConfigReader } from '@backstage/config'; +import { Config } from '@backstage/config'; import { Logger } from 'winston'; import { CatalogRulesEnforcer } from './CatalogRules'; -import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor'; -import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; -import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; -import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor'; -import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; -import { FileReaderProcessor } from './processors/FileReaderProcessor'; -import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; -import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; -import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; -import { LdapOrgReaderProcessor } from './processors/LdapOrgReaderProcessor'; -import { LocationRefProcessor } from './processors/LocationEntityProcessor'; -import { PlaceholderProcessor } from './processors/PlaceholderProcessor'; import * as result from './processors/results'; -import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; import { LocationProcessor, LocationProcessorDataResult, @@ -49,8 +33,6 @@ import { LocationProcessorLocationResult, LocationProcessorResult, } from './processors/types'; -import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; -import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; // The max amount of nesting depth of generated work items @@ -58,94 +40,25 @@ const MAX_DEPTH = 10; type Options = { reader: UrlReader; - logger?: Logger; - config?: Config; - processors?: LocationProcessor[]; + logger: Logger; + config: Config; + processors: LocationProcessor[]; + rulesEnforcer: CatalogRulesEnforcer; }; /** * Implements the reading of a location through a series of processor tasks. */ export class LocationReaders implements LocationReader { - private readonly logger: Logger; - private readonly processors: LocationProcessor[]; - private readonly rulesEnforcer: CatalogRulesEnforcer; + private readonly options: Options; - static defaultProcessors(options: { - logger: Logger; - reader: UrlReader; - config?: Config; - entityPolicy?: EntityPolicy; - }): LocationProcessor[] { - const { - logger, - config = new ConfigReader({}, 'missing-config'), - entityPolicy = new EntityPolicies(), - } = options; - - // TODO(Rugvip): These are added for backwards compatibility if config exists - // The idea is to have everyone migrate from using the old processors to the new - // integration config driven UrlReaders. In an upcoming release we can then completely - // remove support for the old processors, but still keep handling the deprecated location - // types for a while, but with a warning. - const oldProcessors = []; - const pc = config.getOptionalConfig('catalog.processors'); - if (pc?.has('github')) { - logger.warn( - `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, - ); - oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); - } - if (pc?.has('gitlabApi')) { - logger.warn( - `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, - ); - oldProcessors.push(new GitlabApiReaderProcessor(config)); - oldProcessors.push(new GitlabReaderProcessor()); - } - if (pc?.has('bitbucketApi')) { - logger.warn( - `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, - ); - oldProcessors.push(new BitbucketApiReaderProcessor(config)); - } - if (pc?.has('azureApi')) { - logger.warn( - `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, - ); - oldProcessors.push(new AzureApiReaderProcessor(config)); - } - - return [ - StaticLocationProcessor.fromConfig(config), - new FileReaderProcessor(), - ...oldProcessors, - GithubOrgReaderProcessor.fromConfig(config, { logger }), - LdapOrgReaderProcessor.fromConfig(config, { logger }), - new UrlReaderProcessor(options), - new YamlProcessor(), - PlaceholderProcessor.default({ reader: options.reader }), - new CodeOwnersProcessor({ reader: options.reader }), - new EntityPolicyProcessor(entityPolicy), - new LocationRefProcessor(), - new AnnotateLocationEntityProcessor(), - ]; - } - - constructor({ - logger = getVoidLogger(), - config, - reader, - processors = LocationReaders.defaultProcessors({ logger, reader, config }), - }: Options) { - this.logger = logger; - this.processors = processors; - this.rulesEnforcer = config - ? CatalogRulesEnforcer.fromConfig(config) - : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules); + constructor(options: Options) { + this.options = options; } async read(location: LocationSpec): Promise { + const { rulesEnforcer, logger } = this.options; + const output: ReadLocationResult = { entities: [], errors: [] }; let items: LocationProcessorResult[] = [result.location(location, false)]; @@ -159,7 +72,7 @@ export class LocationReaders implements LocationReader { } else if (item.type === 'data') { await this.handleData(item, emit); } else if (item.type === 'entity') { - if (this.rulesEnforcer.isAllowed(item.entity, item.location)) { + if (rulesEnforcer.isAllowed(item.entity, item.location)) { const entity = await this.handleEntity(item, emit); output.entities.push({ entity, @@ -190,7 +103,7 @@ export class LocationReaders implements LocationReader { } const message = `Max recursion depth ${MAX_DEPTH} reached for ${location.type} ${location.target}`; - this.logger.warn(message); + logger.warn(message); output.errors.push({ location, error: new Error(message) }); return output; } @@ -199,7 +112,9 @@ export class LocationReaders implements LocationReader { item: LocationProcessorLocationResult, emit: LocationProcessorEmit, ) { - for (const processor of this.processors) { + const { processors, logger } = this.options; + + for (const processor of processors) { if (processor.readLocation) { try { if ( @@ -210,21 +125,23 @@ export class LocationReaders implements LocationReader { } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); - this.logger.warn(message); + logger.warn(message); } } } const message = `No processor was able to read location ${item.location.type} ${item.location.target}`; emit(result.inputError(item.location, message)); - this.logger.warn(message); + logger.warn(message); } private async handleData( item: LocationProcessorDataResult, emit: LocationProcessorEmit, ) { - for (const processor of this.processors) { + const { processors, logger } = this.options; + + for (const processor of processors) { if (processor.parseData) { try { if (await processor.parseData(item.data, item.location, emit)) { @@ -233,7 +150,7 @@ export class LocationReaders implements LocationReader { } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); - this.logger.warn(message); + logger.warn(message); } } } @@ -246,9 +163,11 @@ export class LocationReaders implements LocationReader { item: LocationProcessorEntityResult, emit: LocationProcessorEmit, ): Promise { + const { processors, logger } = this.options; + let current = item.entity; - for (const processor of this.processors) { + for (const processor of processors) { if (processor.processEntity) { try { current = await processor.processEntity(current, item.location, emit); @@ -261,7 +180,7 @@ export class LocationReaders implements LocationReader { const name = !current.metadata ? '' : current.metadata.name; const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); - this.logger.warn(message); + logger.warn(message); } } } @@ -273,18 +192,20 @@ export class LocationReaders implements LocationReader { item: LocationProcessorErrorResult, emit: LocationProcessorEmit, ) { - this.logger.debug( + const { processors, logger } = this.options; + + logger.debug( `Encountered error at location ${item.location.type} ${item.location.target}, ${item.error}`, ); - for (const processor of this.processors) { + for (const processor of processors) { if (processor.handleError) { try { await processor.handleError(item.error, item.location, emit); } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); - this.logger.warn(message); + logger.warn(message); } } } diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index c196458e2a..294cb10f29 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -21,6 +21,7 @@ import { PlaceholderResolver, ResolverParams, ResolverRead, + textPlaceholderResolver, yamlPlaceholderResolver, } from './PlaceholderProcessor'; @@ -134,9 +135,12 @@ describe('PlaceholderProcessor', () => { expect(read).not.toBeCalled(); }); - it('has builtin text support', async () => { + it('works with the text resolver', async () => { read.mockResolvedValue(Buffer.from('TEXT', 'utf-8')); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { text: textPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( @@ -163,11 +167,14 @@ describe('PlaceholderProcessor', () => { ); }); - it('has builtin json support', async () => { + it('works with the json resolver', async () => { read.mockResolvedValue( Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'), ); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { json: jsonPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( @@ -194,9 +201,12 @@ describe('PlaceholderProcessor', () => { ); }); - it('has builtin yaml support', async () => { + it('works with the yaml resolver', async () => { read.mockResolvedValue(Buffer.from('foo:\n - bar: 7', 'utf-8')); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { yaml: yamlPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( @@ -225,7 +235,10 @@ describe('PlaceholderProcessor', () => { it('resolves absolute path for absolute location', async () => { read.mockResolvedValue(Buffer.from('TEXT', 'utf-8')); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { text: textPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( @@ -258,7 +271,10 @@ describe('PlaceholderProcessor', () => { it('resolves absolute path for relative file location', async () => { read.mockResolvedValue(Buffer.from('TEXT', 'utf-8')); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { text: textPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( @@ -291,10 +307,13 @@ describe('PlaceholderProcessor', () => { it('not resolves relative file path for relative file location', async () => { // We explicitly don't support this case, as it would allow for file system - // traversel attacks. If we want to implement this, we need to have additional + // traversal attacks. If we want to implement this, we need to have additional // security measures in place! read.mockResolvedValue(Buffer.from('TEXT', 'utf-8')); - const processor = PlaceholderProcessor.default({ reader }); + const processor = new PlaceholderProcessor({ + resolvers: { text: textPlaceholderResolver }, + reader, + }); await expect( processor.processEntity( diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts index 91257b68a4..630c78d600 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -43,17 +43,6 @@ type Options = { * that it then fills in with actual data. */ export class PlaceholderProcessor implements LocationProcessor { - static default({ reader }: { reader: UrlReader }) { - return new PlaceholderProcessor({ - resolvers: { - json: jsonPlaceholderResolver, - yaml: yamlPlaceholderResolver, - text: textPlaceholderResolver, - }, - reader, - }); - } - constructor(private readonly options: Options) {} async processEntity(entity: Entity, location: LocationSpec): Promise { diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 79ae55fae1..860f9ff752 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -20,6 +20,11 @@ import yaml from 'yaml'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +/** + * Handles incoming raw data buffers, and if they have a yaml extension, + * attempts to parse them into structured data and emitting them as un- + * validated entities. + */ export class YamlProcessor implements LocationProcessor { async parseData( data: Buffer, diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index c9453d0247..f966ef3348 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -31,6 +31,7 @@ export { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; export { GitlabReaderProcessor } from './GitlabReaderProcessor'; export { LocationRefProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; +export type { PlaceholderResolver } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; export { UrlReaderProcessor } from './UrlReaderProcessor'; export { YamlProcessor } from './YamlProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts new file mode 100644 index 0000000000..e2467654d8 --- /dev/null +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -0,0 +1,139 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { DatabaseManager } from '../database'; +import { LocationProcessorEmit } from '../ingestion'; +import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder'; +import * as result from '../ingestion/processors/results'; + +describe('CatalogBuilder', () => { + const db = DatabaseManager.createTestDatabaseConnection(); + const reader: jest.Mocked = { read: jest.fn() }; + const env: CatalogEnvironment = { + logger: getVoidLogger(), + database: { getClient: () => db }, + config: ConfigReader.fromConfigs([]), + reader, + }; + + afterEach(() => jest.resetAllMocks()); + + it('works with no changes', async () => { + const builder = new CatalogBuilder(env); + const built = await builder.build(); + await expect(built.entitiesCatalog.entities()).resolves.toEqual([]); + await expect(built.locationsCatalog.locations()).resolves.toEqual([ + expect.objectContaining({ + data: expect.objectContaining({ type: 'bootstrap' }), + }), + ]); + }); + + it('works with everything replaced', async () => { + reader.read.mockResolvedValue(Buffer.from('junk')); + + const builder = new CatalogBuilder(env) + .replaceReaderProcessors([ + { + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ) { + expect(location.type).toBe('test'); + emit(result.data(location, await reader.read('ignored'))); + return true; + }, + }, + ]) + .replaceParserProcessors([ + { + async parseData( + data: Buffer, + location: LocationSpec, + emit: LocationProcessorEmit, + ) { + expect(data.toString()).toEqual('junk'); + emit( + result.entity(location, { + apiVersion: 'av', + kind: 'Component', + metadata: { name: 'n' }, + }), + ); + return true; + }, + }, + ]) + .replacePreProcessors([ + { + async processEntity(entity: Entity) { + expect(entity.apiVersion).toBe('av'); + return { + ...entity, + metadata: { ...entity.metadata, namespace: 'ns' }, + }; + }, + }, + ]) + .replaceEntityPolicies([ + { + async enforce(entity: Entity) { + expect(entity.metadata.namespace).toBe('ns'); + return entity; + }, + }, + ]) + .replaceEntityKinds([ + { + async enforce(entity: Entity) { + expect(entity.metadata.namespace).toBe('ns'); + return entity; + }, + }, + ]) + .replacePostProcessors([ + { + async processEntity(entity: Entity) { + return { + ...entity, + metadata: { ...entity.metadata, post: 'p' }, + }; + }, + }, + ]); + const out = await builder.build(); + + const added = await out.higherOrderOperation.addLocation({ + type: 'test', + target: '', + }); + expect(added.entities).toEqual([ + { + apiVersion: 'av', + kind: 'Component', + metadata: expect.objectContaining({ + name: 'n', + namespace: 'ns', + post: 'p', + }), + }, + ]); + }); +}); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts new file mode 100644 index 0000000000..e02005b817 --- /dev/null +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -0,0 +1,523 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; +import { + apiEntityV1alpha1Policy, + componentEntityV1alpha1Policy, + DefaultNamespaceEntityPolicy, + EntityPolicies, + EntityPolicy, + FieldFormatEntityPolicy, + groupEntityV1alpha1Policy, + locationEntityV1alpha1Policy, + makeValidator, + NoForeignRootFieldsEntityPolicy, + ReservedFieldsEntityPolicy, + SchemaValidEntityPolicy, + templateEntityV1alpha1Policy, + userEntityV1alpha1Policy, + Validators, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import lodash from 'lodash'; +import { Logger } from 'winston'; +import { + DatabaseEntitiesCatalog, + DatabaseLocationsCatalog, + EntitiesCatalog, + LocationsCatalog, +} from '../catalog'; +import { DatabaseManager } from '../database'; +import { + AnnotateLocationEntityProcessor, + AzureApiReaderProcessor, + BitbucketApiReaderProcessor, + CodeOwnersProcessor, + EntityPolicyProcessor, + FileReaderProcessor, + GithubOrgReaderProcessor, + GithubReaderProcessor, + GitlabApiReaderProcessor, + GitlabReaderProcessor, + HigherOrderOperation, + HigherOrderOperations, + LocationProcessor, + LocationReaders, + LocationRefProcessor, + PlaceholderProcessor, + PlaceholderResolver, + StaticLocationProcessor, + UrlReaderProcessor, + YamlProcessor, +} from '../ingestion'; +import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; +import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor'; +import { + jsonPlaceholderResolver, + textPlaceholderResolver, + yamlPlaceholderResolver, +} from '../ingestion/processors/PlaceholderProcessor'; + +export type CatalogEnvironment = { + logger: Logger; + database: PluginDatabaseManager; + config: Config; + reader: UrlReader; +}; + +/** + * A builder that helps wire up all of the component parts of the catalog. + * + * The touch points where you can replace or extend behavior are as follows: + * + * - Reader processors can be added or replaced. These implement the + * functionality of reading raw data from a location, in the form of an + * entity definition file. + * - Parser processors can be added or replaced. These accept the raw data as + * read by the previous processors and parse it into raw structured data + * (for example, from a binary buffer containing yaml text, to a JS object + * structure). + * - Placeholder resolvers can be replaced or added. These run on the raw + * structured data between the parsing and pre-processing steps, to replace + * dollar-prefixed entries with their actual values (like $file). + * - Pre-processors can be added or replaced. These take the raw unvalidated + * data from the parser processors and can enrich or extend it before + * validation. This is the place where for example codeowners data can be + * injected into partial entity definitions. + * - Entity policies can be added or replaced. These are the first line of + * validation from the output of the pre-processing step. All policies are + * given the chance to inspect the entity, and all of them have to pass in + * order for the entity to be considered valid from an overall point of + * view. + * - Field format validators can be replaced. These check the format of + * individual core fields such as metadata.name, such that they adhere to + * certain rules. + * - Entity kinds can be added or replaced. These are the second line of + * validation that is applied after the entity policies, which add additional + * kind-specific validation (usually based on a schema). Only one of the + * entity kinds has to accept the entity, but if none of them do, the + * entity is rejected as a whole. + * - Post-processors can be added or replaced. These take the validated + * entities out of the validation step and can perform additional actions on + * them. + */ +export class CatalogBuilder { + private readonly env: CatalogEnvironment; + private entityPolicies: EntityPolicy[]; + private entityPoliciesReplace: boolean; + private entityKinds: EntityPolicy[]; + private entityKindsReplace: boolean; + private readerProcessors: LocationProcessor[]; + private readerProcessorsReplace: boolean; + private parserProcessors: LocationProcessor[]; + private parserProcessorsReplace: boolean; + private preProcessors: LocationProcessor[]; + private preProcessorsReplace: boolean; + private postProcessors: LocationProcessor[]; + private postProcessorsReplace: boolean; + private placeholderResolvers: Record; + private fieldFormatValidators: Partial; + + constructor(env: CatalogEnvironment) { + this.env = env; + this.entityPolicies = []; + this.entityPoliciesReplace = false; + this.entityKinds = []; + this.entityKindsReplace = false; + this.readerProcessors = []; + this.readerProcessorsReplace = false; + this.parserProcessors = []; + this.parserProcessorsReplace = false; + this.preProcessors = []; + this.preProcessorsReplace = false; + this.postProcessors = []; + this.postProcessorsReplace = false; + this.placeholderResolvers = {}; + this.fieldFormatValidators = {}; + } + + /** + * Adds policies that are used to validate entities between the pre- + * processing and post-processing stages. All such policies must pass for the + * entity to be considered valid. + * + * If what you want to do is to replace the rules for what format is allowed + * in various core entity fields (such as metadata.name), you may want to use + * {@link CatalogBuilder#setFieldFormatValidators} instead. + * + * @param policies One or more policies + */ + addEntityPolicy(...policies: EntityPolicy[]): CatalogBuilder { + this.entityPolicies.push(...policies); + return this; + } + + /** + * Sets what policies to use for validation of entities between the pre- + * processing and post-processing stages. All such policies must pass for the + * entity to be considered valid. + * + * If what you want to do is to replace the rules for what format is allowed + * in various core entity fields (such as metadata.name), you may want to use + * {@link CatalogBuilder#setFieldFormatValidators} instead. + * + * This function replaces the default set of policies; use with care. + * + * @param policies One or more policies + */ + replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder { + this.entityPolicies = [...policies]; + this.entityPoliciesReplace = true; + return this; + } + + /** + * Adds entity kinds that are used to validate a certain apiVersion/kind. One + * of the entity kind policies must match a given entity for it to be + * considered valid. + * + * @param policies One or more policies + */ + addEntityKind(...policies: EntityPolicy[]): CatalogBuilder { + this.entityKinds.push(...policies); + return this; + } + + /** + * Sets what entity policies that are used to validate a certain apiVersion/ + * kind. One of the entity kind policies must match a given entity for it to + * be considered valid. + * + * This function replaces the default set of kinds; use with care. + * + * @param policies One or more policies + */ + replaceEntityKinds(policies: EntityPolicy[]): CatalogBuilder { + this.entityKinds = [...policies]; + this.entityKindsReplace = true; + return this; + } + + /** + * Adds processors that support reading of definition files. These are run + * before the entities are parsed, pre-processed, validated and post- + * processed. + * + * @param processors One or more processors + */ + addReaderProcessor(...processors: LocationProcessor[]): CatalogBuilder { + this.readerProcessors.push(...processors); + return this; + } + + /** + * Sets what processors to use for the reading of definition files. These are + * run before the entities are parsed, pre-processed, validated and post- + * processed. + * + * This function replaces the default set of processors in this stage; use + * with care. + * + * @param processors One or more processors + */ + replaceReaderProcessors(processors: LocationProcessor[]): CatalogBuilder { + this.readerProcessors = [...processors]; + this.readerProcessorsReplace = true; + return this; + } + + /** + * Adds processors that run after each definition file has been read, in + * order to parse the raw data. These are run before the entities are + * pre-processed, validated and post-processed. + * + * @param processors One or more processors + */ + addParserProcessor(...processors: LocationProcessor[]): CatalogBuilder { + this.parserProcessors.push(...processors); + return this; + } + + /** + * Sets what processors to run after each definition file has been read, in + * order to parse the raw data. These are run before the entities are + * pre-processed, validated and post-processed. + * + * This function replaces the default set of processors in this stage; use + * with care. + * + * @param processors One or more processors + */ + replaceParserProcessors(processors: LocationProcessor[]): CatalogBuilder { + this.parserProcessors = [...processors]; + this.parserProcessorsReplace = true; + return this; + } + + /** + * Adds processors that run after each entity has been read and parsed, + * but before being validated and post-processed. + * + * @param processors One or more processors + */ + addPreProcessor(...processors: LocationProcessor[]): CatalogBuilder { + this.preProcessors.push(...processors); + return this; + } + + /** + * Sets what processors to run after each entity has been read and parsed, + * but before being validated and post-processed. + * + * This function replaces the default set of processors in this stage; use + * with care. + * + * @param processors One or more processors + */ + replacePreProcessors(processors: LocationProcessor[]): CatalogBuilder { + this.preProcessors = [...processors]; + this.preProcessorsReplace = true; + return this; + } + + /** + * Adds processors that run after each entity has been read, parsed, + * run through the pre-processors, and validated. + * + * @param processors One or more processors + */ + addPostProcessor(...processors: LocationProcessor[]): CatalogBuilder { + this.postProcessors.push(...processors); + return this; + } + + /** + * Sets what processors to run after each entity has been read, parsed, + * run through the pre-processors, and validated. + * + * This function replaces the default set of processors in this stage; use + * with care. + * + * @param processors One or more processors + */ + replacePostProcessors(processors: LocationProcessor[]): CatalogBuilder { + this.postProcessors = [...processors]; + this.postProcessorsReplace = true; + return this; + } + + /** + * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity + * definition files. + * + * @param key The key that identifies the placeholder, e.g. "file" + * @param resolver The resolver that gets values for this placeholder + */ + setPlaceholderResolver(key: string, resolver: PlaceholderResolver) { + this.placeholderResolvers[key] = resolver; + } + + /** + * Sets the validator function to use for one or more special fields of an + * entity. This is useful if the default rules for formatting of fields are + * not sufficient. + * + * This function has no effect if used together with + * {@link CatalogBuilder#replaceEntityPolicies}. + * + * @param validators The (subset of) validators to set + */ + setFieldFormatValidators(validators: Partial) { + lodash.merge(this.fieldFormatValidators, validators); + } + + /** + * Wires up and returns all of the component parts of the catalog + */ + async build(): Promise<{ + entitiesCatalog: EntitiesCatalog; + locationsCatalog: LocationsCatalog; + higherOrderOperation: HigherOrderOperation; + }> { + const { config, database, logger } = this.env; + + const entityPolicy = this.buildEntityPolicy(); + const processors = this.buildProcessors(entityPolicy); + const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config); + + const locationReader = new LocationReaders({ + ...this.env, + processors, + rulesEnforcer, + }); + + const db = await DatabaseManager.createDatabase( + await database.getClient(), + { logger }, + ); + + const entitiesCatalog = new DatabaseEntitiesCatalog(db); + const locationsCatalog = new DatabaseLocationsCatalog(db); + const higherOrderOperation = new HigherOrderOperations( + entitiesCatalog, + locationsCatalog, + locationReader, + logger, + ); + + return { + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + }; + } + + private buildEntityPolicy(): EntityPolicy { + const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace + ? [new SchemaValidEntityPolicy(), ...this.entityPolicies] + : [ + new SchemaValidEntityPolicy(), + new DefaultNamespaceEntityPolicy(), + new NoForeignRootFieldsEntityPolicy(), + new FieldFormatEntityPolicy( + makeValidator(this.fieldFormatValidators), + ), + new ReservedFieldsEntityPolicy(), + ...this.entityPolicies, + ]; + + const entityKinds: EntityPolicy[] = this.entityKindsReplace + ? this.entityKinds + : [ + componentEntityV1alpha1Policy, + groupEntityV1alpha1Policy, + userEntityV1alpha1Policy, + locationEntityV1alpha1Policy, + templateEntityV1alpha1Policy, + apiEntityV1alpha1Policy, + ...this.entityKinds, + ]; + + return EntityPolicies.allOf([ + EntityPolicies.allOf(entityPolicies), + EntityPolicies.oneOf(entityKinds), + ]); + } + + private buildProcessors(entityPolicy: EntityPolicy): LocationProcessor[] { + const { config, reader } = this.env; + + const placeholderResolvers = lodash.merge( + { + json: jsonPlaceholderResolver, + yaml: yamlPlaceholderResolver, + text: textPlaceholderResolver, + }, + this.placeholderResolvers, + ); + + return [ + StaticLocationProcessor.fromConfig(config), + ...this.buildReaderProcessors(), + ...this.buildParserProcessors(), + new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), + ...this.buildPreProcessors(), + new EntityPolicyProcessor(entityPolicy), + ...this.buildPostProcessors(), + ]; + } + + private buildReaderProcessors(): LocationProcessor[] { + const { config, logger, reader } = this.env; + + if (this.readerProcessorsReplace) { + return this.readerProcessors; + } + + // TODO(Rugvip): These are added for backwards compatibility if config exists + // The idea is to have everyone migrate from using the old processors to the new + // integration config driven UrlReaders. In an upcoming release we can then completely + // remove support for the old processors, but still keep handling the deprecated location + // types for a while, but with a warning. + const oldProcessors = []; + const pc = config.getOptionalConfig('catalog.processors'); + if (pc?.has('github')) { + logger.warn( + `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, + ); + oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); + } + if (pc?.has('gitlabApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, + ); + oldProcessors.push(new GitlabApiReaderProcessor(config)); + oldProcessors.push(new GitlabReaderProcessor()); + } + if (pc?.has('bitbucketApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, + ); + oldProcessors.push(new BitbucketApiReaderProcessor(config)); + } + if (pc?.has('azureApi')) { + logger.warn( + `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, + ); + oldProcessors.push(new AzureApiReaderProcessor(config)); + } + + return [ + new FileReaderProcessor(), + ...oldProcessors, + GithubOrgReaderProcessor.fromConfig(config, { logger }), + LdapOrgReaderProcessor.fromConfig(config, { logger }), + new UrlReaderProcessor({ reader, logger }), + ...this.readerProcessors, + ]; + } + + private buildParserProcessors(): LocationProcessor[] { + if (this.parserProcessorsReplace) { + return this.parserProcessors; + } + + return [new YamlProcessor(), ...this.parserProcessors]; + } + + private buildPreProcessors(): LocationProcessor[] { + const { reader } = this.env; + + if (this.preProcessorsReplace) { + return this.preProcessors; + } + + return [new CodeOwnersProcessor({ reader }), ...this.preProcessors]; + } + + private buildPostProcessors(): LocationProcessor[] { + if (this.postProcessorsReplace) { + return this.postProcessors; + } + + return [ + new LocationRefProcessor(), + new AnnotateLocationEntityProcessor(), + ...this.postProcessors, + ]; + } +} diff --git a/plugins/catalog-backend/src/service/index.ts b/plugins/catalog-backend/src/service/index.ts new file mode 100644 index 0000000000..c6da8db486 --- /dev/null +++ b/plugins/catalog-backend/src/service/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CatalogBuilder } from './CatalogBuilder'; +export { createRouter } from './router'; +export type { RouterOptions } from './router'; diff --git a/plugins/catalog-backend/src/service/standaloneServer.ts b/plugins/catalog-backend/src/service/standaloneServer.ts index 5939c7eb58..713a5783ba 100644 --- a/plugins/catalog-backend/src/service/standaloneServer.ts +++ b/plugins/catalog-backend/src/service/standaloneServer.ts @@ -18,16 +18,14 @@ import { createServiceBuilder, loadBackendConfig, UrlReaders, + useHotMemoize, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { Server } from 'http'; import { Logger } from 'winston'; -import { HigherOrderOperations } from '..'; -import { DatabaseEntitiesCatalog } from '../catalog/DatabaseEntitiesCatalog'; -import { DatabaseLocationsCatalog } from '../catalog/DatabaseLocationsCatalog'; -import { DatabaseManager } from '../database/DatabaseManager'; +import { DatabaseManager } from '../database'; +import { CatalogBuilder } from './CatalogBuilder'; import { createRouter } from './router'; -import { LocationReaders } from '../ingestion'; export interface ServerOptions { port: number; @@ -41,18 +39,22 @@ export async function startStandaloneServer( const logger = options.logger.child({ service: 'catalog-backend' }); const config = ConfigReader.fromConfigs(await loadBackendConfig()); const reader = UrlReaders.default({ logger, config }); + const db = useHotMemoize(module, () => + DatabaseManager.createInMemoryDatabaseConnection(), + ); logger.debug('Creating application...'); - const db = await DatabaseManager.createInMemoryDatabase({ logger }); - const entitiesCatalog = new DatabaseEntitiesCatalog(db); - const locationsCatalog = new DatabaseLocationsCatalog(db); - const locationReader = new LocationReaders({ logger, config, reader }); - const higherOrderOperation = new HigherOrderOperations( + const builder = new CatalogBuilder({ + logger, + database: { getClient: () => db }, + config, + reader, + }); + const { entitiesCatalog, locationsCatalog, - locationReader, - logger, - ); + higherOrderOperation, + } = await builder.build(); logger.debug('Starting application server...'); const router = await createRouter({