From 4036ff59d01de97f08eff7f7b03e2ab0e6aead92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 18 Oct 2020 21:16:35 +0200 Subject: [PATCH 01/25] feat(catalog-backend): First steps of refactoring - split processEntity --- .changeset/carpal-tunnel-driver.md | 13 + .../src/ingestion/LocationReaders.ts | 72 +++- .../AnnotateLocationEntityProcessor.ts | 5 +- .../processors/CodeOwnersProcessor.test.ts | 8 +- .../processors/CodeOwnersProcessor.ts | 5 +- .../processors/EntityPolicyProcessor.ts | 34 -- .../processors/LocationEntityProcessor.ts | 2 +- .../processors/PlaceholderProcessor.test.ts | 20 +- .../processors/PlaceholderProcessor.ts | 5 +- .../src/ingestion/processors/index.ts | 1 - .../src/ingestion/processors/types.ts | 26 +- .../src/service/CatalogBuilder.test.ts | 91 ++--- .../src/service/CatalogBuilder.ts | 316 +++++------------- 13 files changed, 257 insertions(+), 341 deletions(-) create mode 100644 .changeset/carpal-tunnel-driver.md delete mode 100644 plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts diff --git a/.changeset/carpal-tunnel-driver.md b/.changeset/carpal-tunnel-driver.md new file mode 100644 index 0000000000..2e6a814533 --- /dev/null +++ b/.changeset/carpal-tunnel-driver.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +- The `CatalogProcessor` API was updated to have `preProcessEntity` and + `postProcessEntity` methods, instead of just one `processEntity`. This makes + it easier to make processors that have several stages in one, and to make + different processors more position independent in the list of processors. +- The `EntityPolicy` is now given directly to the `LocationReaders`, instead of + being enforced inside a policy. We have decided to separate out the act of + validating an entity to be outside of the processing flow, to make it + possible to apply more liberally and to evolve it as a separate concept. +- Because of the above, the `EntityPolicyProcessor` has been removed. diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index c36b3d2b05..e23a4524b6 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -17,6 +17,7 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity, + EntityPolicy, ENTITY_DEFAULT_NAMESPACE, LocationSpec, } from '@backstage/catalog-model'; @@ -44,6 +45,7 @@ type Options = { config: Config; processors: CatalogProcessor[]; rulesEnforcer: CatalogRulesEnforcer; + policy: EntityPolicy; }; /** @@ -74,10 +76,12 @@ export class LocationReaders implements LocationReader { } else if (item.type === 'entity') { if (rulesEnforcer.isAllowed(item.entity, item.location)) { const entity = await this.handleEntity(item, emit); - output.entities.push({ - entity, - location: item.location, - }); + if (entity) { + output.entities.push({ + entity, + location: item.location, + }); + } } else { output.errors.push({ location: item.location, @@ -162,25 +166,65 @@ export class LocationReaders implements LocationReader { private async handleEntity( item: CatalogProcessorEntityResult, emit: CatalogProcessorEmit, - ): Promise { + ): Promise { const { processors, logger } = this.options; let current = item.entity; + // Construct the name carefully, this happens before validation below + // so we do not want to crash here due to missing metadata or so + const kind = current.kind || ''; + const namespace = !current.metadata + ? '' + : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE; + const name = !current.metadata ? '' : current.metadata.name; + for (const processor of processors) { - if (processor.processEntity) { + if (processor.preProcessEntity) { try { - current = await processor.processEntity(current, item.location, emit); + current = await processor.preProcessEntity( + current, + item.location, + emit, + ); } catch (e) { - // Construct the name carefully, if we got validation errors we do - // not want to crash here due to missing metadata or so - const namespace = !current.metadata - ? '' - : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE; - const name = !current.metadata ? '' : current.metadata.name; - const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; + const message = `Processor ${processor.constructor.name} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); logger.warn(message); + return undefined; + } + } + } + + try { + const next = await this.options.policy.enforce(current); + if (!next) { + const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}`; + emit(result.generalError(item.location, message)); + logger.warn(message); + return undefined; + } + current = next; + } catch (e) { + const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; + emit(result.inputError(item.location, message)); + logger.warn(message); + return undefined; + } + + for (const processor of processors) { + if (processor.postProcessEntity) { + try { + current = await processor.postProcessEntity( + current, + item.location, + emit, + ); + } catch (e) { + const message = `Processor ${processor.constructor.name} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; + emit(result.generalError(item.location, message)); + logger.warn(message); + return undefined; } } } diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts index fea70b6c5e..ea8afcddc5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts @@ -19,7 +19,10 @@ import lodash from 'lodash'; import { CatalogProcessor } from './types'; export class AnnotateLocationEntityProcessor implements CatalogProcessor { - async processEntity(entity: Entity, location: LocationSpec): Promise { + async preProcessEntity( + entity: Entity, + location: LocationSpec, + ): Promise { return lodash.merge( { metadata: { diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts index c1c83b7c6c..247020a547 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts @@ -224,7 +224,7 @@ describe(CodeOwnersProcessor, () => { spec: { owner: '@acme/foo-team' }, }); - const result = await processor.processEntity( + const result = await processor.preProcessEntity( entity as any, mockLocation(), ); @@ -235,7 +235,7 @@ describe(CodeOwnersProcessor, () => { it('should ignore url locations', async () => { const { entity, processor } = setupTest(); - const result = await processor.processEntity( + const result = await processor.preProcessEntity( entity as any, mockLocation({ type: 'url' }), ); @@ -246,7 +246,7 @@ describe(CodeOwnersProcessor, () => { it('should ignore invalid kinds', async () => { const { entity, processor } = setupTest({ kind: 'Group' }); - const result = await processor.processEntity( + const result = await processor.preProcessEntity( entity as any, mockLocation(), ); @@ -257,7 +257,7 @@ describe(CodeOwnersProcessor, () => { it('should set owner from codeowner', async () => { const { entity, processor } = setupTest(); - const result = await processor.processEntity( + const result = await processor.preProcessEntity( entity as any, mockLocation(), ); diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts index 1802e01d99..bc78b6e34a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts @@ -40,7 +40,10 @@ type Options = { export class CodeOwnersProcessor implements CatalogProcessor { constructor(private readonly options: Options) {} - async processEntity(entity: Entity, location: LocationSpec): Promise { + async preProcessEntity( + entity: Entity, + location: LocationSpec, + ): Promise { // Only continue if the owner is not set if ( !entity || diff --git a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts deleted file mode 100644 index 87543d91f4..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity, EntityPolicy } from '@backstage/catalog-model'; -import { CatalogProcessor } from './types'; - -export class EntityPolicyProcessor implements CatalogProcessor { - private readonly policy: EntityPolicy; - - constructor(policy: EntityPolicy) { - this.policy = policy; - } - - async processEntity(entity: Entity): Promise { - const output = await this.policy.enforce(entity); - if (!output) { - throw new Error(`Entity did not match any known schema`); - } - return output; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts index 1917f58fbd..86e17f07f2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts @@ -19,7 +19,7 @@ import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class LocationRefProcessor implements CatalogProcessor { - async processEntity( + async postProcessEntity( entity: Entity, _location: LocationSpec, emit: CatalogProcessorEmit, diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index 294cb10f29..44f3163a97 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -46,7 +46,7 @@ describe('PlaceholderProcessor', () => { reader, }); await expect( - processor.processEntity(input, { type: 't', target: 'l' }), + processor.preProcessEntity(input, { type: 't', target: 'l' }), ).resolves.toBe(input); }); @@ -62,7 +62,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -98,7 +98,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -122,7 +122,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -143,7 +143,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -177,7 +177,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -209,7 +209,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -241,7 +241,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -277,7 +277,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', @@ -316,7 +316,7 @@ describe('PlaceholderProcessor', () => { }); await expect( - processor.processEntity( + processor.preProcessEntity( { apiVersion: 'a', kind: 'k', diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts index 3941a2bedc..0e23d51f61 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -45,7 +45,10 @@ type Options = { export class PlaceholderProcessor implements CatalogProcessor { constructor(private readonly options: Options) {} - async processEntity(entity: Entity, location: LocationSpec): Promise { + async preProcessEntity( + entity: Entity, + location: LocationSpec, + ): Promise { const process = async (data: any): Promise<[any, boolean]> => { if (!data || !(data instanceof Object)) { // Scalars can't have placeholders diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index f966ef3348..59a87af555 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -23,7 +23,6 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess export { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; export { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; -export { EntityPolicyProcessor } from './EntityPolicyProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubReaderProcessor } from './GithubReaderProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 0e6b1491a6..c9adf07eff 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -46,15 +46,33 @@ export type CatalogProcessor = { ): Promise; /** - * Processes an emitted entity, e.g. by validating or modifying it. + * Pre-processes an emitted entity, after it has been emitted but before it + * has been validated. * - * @param entity The entity to process + * This type of processing usually involves enriching the entity with + * additional data, and the input entity may actually still be incomplete + * when the processor is invoked. + * + * @param entity The (possibly partial) entity to process * @param location The location that the entity came from - * @param read Reads the contents of a location * @param emit A sink for auxiliary items resulting from the processing * @returns The same entity or a modified version of it */ - processEntity?( + preProcessEntity?( + entity: Entity, + location: LocationSpec, + emit: CatalogProcessorEmit, + ): Promise; + + /** + * Post-processes an emitted entity, after it has been validated. + * + * @param entity The entity to process + * @param location The location that the entity came from + * @param emit A sink for auxiliary items resulting from the processing + * @returns The same entity or a modified version of it + */ + postProcessEntity?( entity: Entity, location: LocationSpec, emit: CatalogProcessorEmit, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 66e7dba682..3f7c613c00 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -49,49 +49,6 @@ describe('CatalogBuilder', () => { reader.read.mockResolvedValue(Buffer.from('junk')); const builder = new CatalogBuilder(env) - .replaceReaderProcessors([ - { - async readLocation( - location: LocationSpec, - _optional: boolean, - emit: CatalogProcessorEmit, - ) { - expect(location.type).toBe('test'); - emit(result.data(location, await reader.read('ignored'))); - return true; - }, - }, - ]) - .replaceParserProcessors([ - { - async parseData( - data: Buffer, - location: LocationSpec, - emit: CatalogProcessorEmit, - ) { - expect(data.toString()).toEqual('junk'); - emit( - result.entity(location, { - apiVersion: 'av', - kind: 'Component', - metadata: { name: 'n' }, - }), - ); - return true; - }, - }, - ]) - .replacePreProcessors([ - { - async processEntity(entity: Entity) { - expect(entity.apiVersion).toBe('av'); - return { - ...entity, - metadata: { ...entity.metadata, namespace: 'ns' }, - }; - }, - }, - ]) .replaceEntityPolicies([ { async enforce(entity: Entity) { @@ -108,9 +65,51 @@ describe('CatalogBuilder', () => { }, }, ]) - .replacePostProcessors([ + .setPlaceholderResolver('t', async ({ value }) => { + expect(value).toBe('tt'); + return 'tt2'; + }) + .setFieldFormatValidators({ + isValidEntityName: n => { + expect(n).toBe('n'); + return true; + }, + }) + .replaceProcessors([ { - async processEntity(entity: Entity) { + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ) { + expect(location.type).toBe('test'); + emit(result.data(location, await reader.read('ignored'))); + return true; + }, + async parseData( + data: Buffer, + location: LocationSpec, + emit: CatalogProcessorEmit, + ) { + expect(data.toString()).toEqual('junk'); + emit( + result.entity(location, { + apiVersion: 'av', + kind: 'Component', + metadata: { name: 'n', replaced: { $t: 'tt' } }, + }), + ); + return true; + }, + async preProcessEntity(entity: Entity) { + expect(entity.apiVersion).toBe('av'); + return { + ...entity, + metadata: { ...entity.metadata, namespace: 'ns' }, + }; + }, + async postProcessEntity(entity: Entity) { + expect(entity.metadata.namespace).toBe('ns'); return { ...entity, metadata: { ...entity.metadata, post: 'p' }, @@ -124,6 +123,7 @@ describe('CatalogBuilder', () => { type: 'test', target: '', }); + expect.assertions(8); expect(added.entities).toEqual([ { apiVersion: 'av', @@ -132,6 +132,7 @@ describe('CatalogBuilder', () => { name: 'n', namespace: 'ns', post: 'p', + replaced: 'tt2', }), }, ]); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index aa4f97258d..01a4cd2435 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -48,7 +48,6 @@ import { BitbucketApiReaderProcessor, CatalogProcessor, CodeOwnersProcessor, - EntityPolicyProcessor, FileReaderProcessor, GithubOrgReaderProcessor, GithubReaderProcessor, @@ -84,36 +83,24 @@ export type CatalogEnvironment = { * * The touch points where you can replace or extend behavior are as follows: * - * - Reader processors can be added or replaced. These implement the - * functionality of reading raw data from a location, in the form of an - * entity definition file. - * - Parser processors can be added or replaced. These accept the raw data as - * read by the previous processors and parse it into raw structured data - * (for example, from a binary buffer containing yaml text, to a JS object - * structure). + * - Entity policies can be added or replaced. These are automatically run + * after the processors' pre-processing steps. All policies are given the + * chance to inspect the entity, and all of them have to pass in order for + * the entity to be considered valid from an overall point of view. + * - Entity kinds can be added or replaced. These are the second line of + * validation that is applied after the entity policies, which adds + * additional kind-specific validation (usually based on a schema). Only one + * of the entity kinds has to accept the entity, but if none of them do, the + * entity is rejected as a whole. * - Placeholder resolvers can be replaced or added. These run on the raw * structured data between the parsing and pre-processing steps, to replace * dollar-prefixed entries with their actual values (like $file). - * - Pre-processors can be added or replaced. These take the raw unvalidated - * data from the parser processors and can enrich or extend it before - * validation. This is the place where for example codeowners data can be - * injected into partial entity definitions. - * - Entity policies can be added or replaced. These are the first line of - * validation from the output of the pre-processing step. All policies are - * given the chance to inspect the entity, and all of them have to pass in - * order for the entity to be considered valid from an overall point of - * view. * - Field format validators can be replaced. These check the format of - * individual core fields such as metadata.name, such that they adhere to - * certain rules. - * - Entity kinds can be added or replaced. These are the second line of - * validation that is applied after the entity policies, which add additional - * kind-specific validation (usually based on a schema). Only one of the - * entity kinds has to accept the entity, but if none of them do, the - * entity is rejected as a whole. - * - Post-processors can be added or replaced. These take the validated - * entities out of the validation step and can perform additional actions on - * them. + * individual core fields such as metadata.name, to ensure that they adhere + * to certain rules. + * - Processors can be added or replaced. These implement the functionality of + * reading, parsing and processing the entity data before it is persisted in + * the catalog. */ export class CatalogBuilder { private readonly env: CatalogEnvironment; @@ -121,16 +108,10 @@ export class CatalogBuilder { private entityPoliciesReplace: boolean; private entityKinds: EntityPolicy[]; private entityKindsReplace: boolean; - private readerProcessors: CatalogProcessor[]; - private readerProcessorsReplace: boolean; - private parserProcessors: CatalogProcessor[]; - private parserProcessorsReplace: boolean; - private preProcessors: CatalogProcessor[]; - private preProcessorsReplace: boolean; - private postProcessors: CatalogProcessor[]; - private postProcessorsReplace: boolean; private placeholderResolvers: Record; private fieldFormatValidators: Partial; + private processors: CatalogProcessor[]; + private processorsReplace: boolean; constructor(env: CatalogEnvironment) { this.env = env; @@ -138,16 +119,10 @@ export class CatalogBuilder { this.entityPoliciesReplace = false; this.entityKinds = []; this.entityKindsReplace = false; - this.readerProcessors = []; - this.readerProcessorsReplace = false; - this.parserProcessors = []; - this.parserProcessorsReplace = false; - this.preProcessors = []; - this.preProcessorsReplace = false; - this.postProcessors = []; - this.postProcessorsReplace = false; this.placeholderResolvers = {}; this.fieldFormatValidators = {}; + this.processors = []; + this.processorsReplace = false; } /** @@ -212,114 +187,6 @@ export class CatalogBuilder { return this; } - /** - * Adds processors that support reading of definition files. These are run - * before the entities are parsed, pre-processed, validated and post- - * processed. - * - * @param processors One or more processors - */ - addReaderProcessor(...processors: CatalogProcessor[]): CatalogBuilder { - this.readerProcessors.push(...processors); - return this; - } - - /** - * Sets what processors to use for the reading of definition files. These are - * run before the entities are parsed, pre-processed, validated and post- - * processed. - * - * This function replaces the default set of processors in this stage; use - * with care. - * - * @param processors One or more processors - */ - replaceReaderProcessors(processors: CatalogProcessor[]): CatalogBuilder { - this.readerProcessors = [...processors]; - this.readerProcessorsReplace = true; - return this; - } - - /** - * Adds processors that run after each definition file has been read, in - * order to parse the raw data. These are run before the entities are - * pre-processed, validated and post-processed. - * - * @param processors One or more processors - */ - addParserProcessor(...processors: CatalogProcessor[]): CatalogBuilder { - this.parserProcessors.push(...processors); - return this; - } - - /** - * Sets what processors to run after each definition file has been read, in - * order to parse the raw data. These are run before the entities are - * pre-processed, validated and post-processed. - * - * This function replaces the default set of processors in this stage; use - * with care. - * - * @param processors One or more processors - */ - replaceParserProcessors(processors: CatalogProcessor[]): CatalogBuilder { - this.parserProcessors = [...processors]; - this.parserProcessorsReplace = true; - return this; - } - - /** - * Adds processors that run after each entity has been read and parsed, - * but before being validated and post-processed. - * - * @param processors One or more processors - */ - addPreProcessor(...processors: CatalogProcessor[]): CatalogBuilder { - this.preProcessors.push(...processors); - return this; - } - - /** - * Sets what processors to run after each entity has been read and parsed, - * but before being validated and post-processed. - * - * This function replaces the default set of processors in this stage; use - * with care. - * - * @param processors One or more processors - */ - replacePreProcessors(processors: CatalogProcessor[]): CatalogBuilder { - this.preProcessors = [...processors]; - this.preProcessorsReplace = true; - return this; - } - - /** - * Adds processors that run after each entity has been read, parsed, - * run through the pre-processors, and validated. - * - * @param processors One or more processors - */ - addPostProcessor(...processors: CatalogProcessor[]): CatalogBuilder { - this.postProcessors.push(...processors); - return this; - } - - /** - * Sets what processors to run after each entity has been read, parsed, - * run through the pre-processors, and validated. - * - * This function replaces the default set of processors in this stage; use - * with care. - * - * @param processors One or more processors - */ - replacePostProcessors(processors: CatalogProcessor[]): CatalogBuilder { - this.postProcessors = [...processors]; - this.postProcessorsReplace = true; - return this; - } - /** * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity * definition files. @@ -327,8 +194,12 @@ export class CatalogBuilder { * @param key The key that identifies the placeholder, e.g. "file" * @param resolver The resolver that gets values for this placeholder */ - setPlaceholderResolver(key: string, resolver: PlaceholderResolver) { + setPlaceholderResolver( + key: string, + resolver: PlaceholderResolver, + ): CatalogBuilder { this.placeholderResolvers[key] = resolver; + return this; } /** @@ -341,8 +212,34 @@ export class CatalogBuilder { * * @param validators The (subset of) validators to set */ - setFieldFormatValidators(validators: Partial) { + setFieldFormatValidators(validators: Partial): CatalogBuilder { lodash.merge(this.fieldFormatValidators, validators); + return this; + } + + /** + * Adds entity processors. These are responsible for reading, parsing, and + * processing entities before they are persisted in the catalog. + * + * @param processors One or more processors + */ + addProcessor(...processors: CatalogProcessor[]): CatalogBuilder { + this.processors.push(...processors); + return this; + } + + /** + * Sets what entity processors to use. These are responsible for reading, + * parsing, and processing entities before they are persisted in the catalog. + * + * This function replaces the default set of processors; use with care. + * + * @param processors One or more processors + */ + replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder { + this.processors = [...processors]; + this.processorsReplace = true; + return this; } /** @@ -355,14 +252,15 @@ export class CatalogBuilder { }> { const { config, database, logger } = this.env; - const entityPolicy = this.buildEntityPolicy(); - const processors = this.buildProcessors(entityPolicy); + const policy = this.buildEntityPolicy(); + const processors = this.buildProcessors(); const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config); const locationReader = new LocationReaders({ ...this.env, processors, rulesEnforcer, + policy, }); const db = await DatabaseManager.createDatabase( @@ -418,106 +316,74 @@ export class CatalogBuilder { ]); } - private buildProcessors(entityPolicy: EntityPolicy): CatalogProcessor[] { - const { config, reader } = this.env; + private buildProcessors(): CatalogProcessor[] { + const { config, logger, reader } = this.env; - const placeholderResolvers = lodash.merge( - { - json: jsonPlaceholderResolver, - yaml: yamlPlaceholderResolver, - text: textPlaceholderResolver, - }, - this.placeholderResolvers, - ); + const placeholderResolvers: Record = { + json: jsonPlaceholderResolver, + yaml: yamlPlaceholderResolver, + text: textPlaceholderResolver, + ...this.placeholderResolvers, + }; + + const processors = this.processorsReplace + ? this.processors + : [ + new FileReaderProcessor(), + GithubOrgReaderProcessor.fromConfig(config, { logger }), + LdapOrgReaderProcessor.fromConfig(config, { logger }), + new UrlReaderProcessor({ reader, logger }), + new YamlProcessor(), + new CodeOwnersProcessor({ reader }), + new LocationRefProcessor(), + new AnnotateLocationEntityProcessor(), + ]; return [ StaticLocationProcessor.fromConfig(config), - ...this.buildReaderProcessors(), - ...this.buildParserProcessors(), new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }), - ...this.buildPreProcessors(), - new EntityPolicyProcessor(entityPolicy), - ...this.buildPostProcessors(), + ...this.buildDeprecatedReaderProcessors(), + ...processors, ]; } - private buildReaderProcessors(): CatalogProcessor[] { - const { config, logger, reader } = this.env; + // TODO(Rugvip): These are added for backwards compatibility if config exists + // The idea is to have everyone migrate from using the old processors to + // the new integration config driven UrlReaders. In an upcoming release we + // can then completely remove support for the old processors, but still + // keep handling the deprecated location types for a while, but with a + // warning. + private buildDeprecatedReaderProcessors(): CatalogProcessor[] { + const { config, logger } = this.env; - if (this.readerProcessorsReplace) { - return this.readerProcessors; - } - - // TODO(Rugvip): These are added for backwards compatibility if config exists - // The idea is to have everyone migrate from using the old processors to the new - // integration config driven UrlReaders. In an upcoming release we can then completely - // remove support for the old processors, but still keep handling the deprecated location - // types for a while, but with a warning. - const oldProcessors = []; + const result = []; const pc = config.getOptionalConfig('catalog.processors'); if (pc?.has('github')) { logger.warn( `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`, ); - oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger)); + result.push(GithubReaderProcessor.fromConfig(config, logger)); } if (pc?.has('gitlabApi')) { logger.warn( `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`, ); - oldProcessors.push(new GitlabApiReaderProcessor(config)); - oldProcessors.push(new GitlabReaderProcessor()); + result.push(new GitlabApiReaderProcessor(config)); + result.push(new GitlabReaderProcessor()); } if (pc?.has('bitbucketApi')) { logger.warn( `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`, ); - oldProcessors.push(new BitbucketApiReaderProcessor(config)); + result.push(new BitbucketApiReaderProcessor(config)); } if (pc?.has('azureApi')) { logger.warn( `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`, ); - oldProcessors.push(new AzureApiReaderProcessor(config)); + result.push(new AzureApiReaderProcessor(config)); } - return [ - new FileReaderProcessor(), - ...oldProcessors, - GithubOrgReaderProcessor.fromConfig(config, { logger }), - LdapOrgReaderProcessor.fromConfig(config, { logger }), - new UrlReaderProcessor({ reader, logger }), - ...this.readerProcessors, - ]; - } - - private buildParserProcessors(): CatalogProcessor[] { - if (this.parserProcessorsReplace) { - return this.parserProcessors; - } - - return [new YamlProcessor(), ...this.parserProcessors]; - } - - private buildPreProcessors(): CatalogProcessor[] { - const { reader } = this.env; - - if (this.preProcessorsReplace) { - return this.preProcessors; - } - - return [new CodeOwnersProcessor({ reader }), ...this.preProcessors]; - } - - private buildPostProcessors(): CatalogProcessor[] { - if (this.postProcessorsReplace) { - return this.postProcessors; - } - - return [ - new LocationRefProcessor(), - new AnnotateLocationEntityProcessor(), - ...this.postProcessors, - ]; + return result; } } From 6480278da9a0180648beba0157021eff1804c85c Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Thu, 15 Oct 2020 16:53:35 +0200 Subject: [PATCH 02/25] change jsx to tsx --- .../components/Button/{Button.test.jsx => Button.test.tsx} | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename packages/core/src/components/Button/{Button.test.jsx => Button.test.tsx} (94%) diff --git a/packages/core/src/components/Button/Button.test.jsx b/packages/core/src/components/Button/Button.test.tsx similarity index 94% rename from packages/core/src/components/Button/Button.test.jsx rename to packages/core/src/components/Button/Button.test.tsx index 115835be3f..8bae5f2767 100644 --- a/packages/core/src/components/Button/Button.test.jsx +++ b/packages/core/src/components/Button/Button.test.tsx @@ -34,7 +34,9 @@ describe('