paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases Wiki Activity

feat(catalog-backend): First steps of refactoring - split processEntity

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-10-18 21:16:35 +02:00
parent 2e1a0b8c99
commit 4036ff59d0
13 changed files with 257 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/carpal-tunnel-driver.md
+13
View File
@@ -0,0 +1,13 @@
---
'@backstage/plugin-catalog-backend': minor
---
- The `CatalogProcessor` API was updated to have `preProcessEntity` and
`postProcessEntity` methods, instead of just one `processEntity`. This makes
it easier to make processors that have several stages in one, and to make
different processors more position independent in the list of processors.
- The `EntityPolicy` is now given directly to the `LocationReaders`, instead of
being enforced inside a policy. We have decided to separate out the act of
validating an entity to be outside of the processing flow, to make it
possible to apply more liberally and to evolve it as a separate concept.
- Because of the above, the `EntityPolicyProcessor` has been removed.
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+58 -14
View File
@@ -17,6 +17,7 @@
import { UrlReader } from '@backstage/backend-common';
import {
Entity,
EntityPolicy,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
} from '@backstage/catalog-model';
@@ -44,6 +45,7 @@ type Options = {
config: Config;
processors: CatalogProcessor[];
rulesEnforcer: CatalogRulesEnforcer;
policy: EntityPolicy;
};
/**
@@ -74,10 +76,12 @@ export class LocationReaders implements LocationReader {
} else if (item.type === 'entity') {
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
const entity = await this.handleEntity(item, emit);
output.entities.push({
entity,
location: item.location,
});
if (entity) {
output.entities.push({
entity,
location: item.location,
});
}
} else {
output.errors.push({
location: item.location,
@@ -162,25 +166,65 @@ export class LocationReaders implements LocationReader {
private async handleEntity(
item: CatalogProcessorEntityResult,
emit: CatalogProcessorEmit,
): Promise<Entity> {
): Promise<Entity | undefined> {
const { processors, logger } = this.options;
let current = item.entity;
// Construct the name carefully, this happens before validation below
// so we do not want to crash here due to missing metadata or so
const kind = current.kind || '';
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
for (const processor of processors) {
if (processor.processEntity) {
if (processor.preProcessEntity) {
try {
current = await processor.processEntity(current, item.location, emit);
current = await processor.preProcessEntity(
current,
item.location,
emit,
);
} catch (e) {
// Construct the name carefully, if we got validation errors we do
// not want to crash here due to missing metadata or so
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
const message = `Processor ${processor.constructor.name} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
try {
const next = await this.options.policy.enforce(current);
if (!next) {
const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
current = next;
} catch (e) {
const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
for (const processor of processors) {
if (processor.postProcessEntity) {
try {
current = await processor.postProcessEntity(
current,
item.location,
emit,
);
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
+4 -1
View File
@@ -19,7 +19,10 @@ import lodash from 'lodash';
import { CatalogProcessor } from './types';
export class AnnotateLocationEntityProcessor implements CatalogProcessor {
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
return lodash.merge(
{
metadata: {
plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
+4 -4
View File
@@ -224,7 +224,7 @@ describe(CodeOwnersProcessor, () => {
spec: { owner: '@acme/foo-team' },
});
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
@@ -235,7 +235,7 @@ describe(CodeOwnersProcessor, () => {
it('should ignore url locations', async () => {
const { entity, processor } = setupTest();
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation({ type: 'url' }),
);
@@ -246,7 +246,7 @@ describe(CodeOwnersProcessor, () => {
it('should ignore invalid kinds', async () => {
const { entity, processor } = setupTest({ kind: 'Group' });
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
@@ -257,7 +257,7 @@ describe(CodeOwnersProcessor, () => {
it('should set owner from codeowner', async () => {
const { entity, processor } = setupTest();
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts
+4 -1
View File
@@ -40,7 +40,10 @@ type Options = {
export class CodeOwnersProcessor implements CatalogProcessor {
constructor(private readonly options: Options) {}
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
// Only continue if the owner is not set
if (
!entity ||
plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts
-34
View File
@@ -1,34 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import { CatalogProcessor } from './types';
export class EntityPolicyProcessor implements CatalogProcessor {
private readonly policy: EntityPolicy;
constructor(policy: EntityPolicy) {
this.policy = policy;
}
async processEntity(entity: Entity): Promise<Entity> {
const output = await this.policy.enforce(entity);
if (!output) {
throw new Error(`Entity did not match any known schema`);
}
return output;
}
}
plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts
+1 -1
View File
@@ -19,7 +19,7 @@ import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
export class LocationRefProcessor implements CatalogProcessor {
async processEntity(
async postProcessEntity(
entity: Entity,
_location: LocationSpec,
emit: CatalogProcessorEmit,
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts
+10 -10
View File
@@ -46,7 +46,7 @@ describe('PlaceholderProcessor', () => {
reader,
});
await expect(
processor.processEntity(input, { type: 't', target: 'l' }),
processor.preProcessEntity(input, { type: 't', target: 'l' }),
).resolves.toBe(input);
});
@@ -62,7 +62,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -98,7 +98,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -122,7 +122,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -143,7 +143,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -177,7 +177,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -209,7 +209,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -241,7 +241,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -277,7 +277,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -316,7 +316,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts
+4 -1
View File
@@ -45,7 +45,10 @@ type Options = {
export class PlaceholderProcessor implements CatalogProcessor {
constructor(private readonly options: Options) {}
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
const process = async (data: any): Promise<[any, boolean]> => {
if (!data || !(data instanceof Object)) {
// Scalars can't have placeholders
plugins/catalog-backend/src/ingestion/processors/index.ts
-1
View File
@@ -23,7 +23,6 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess
export { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
export { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
export { CodeOwnersProcessor } from './CodeOwnersProcessor';
export { EntityPolicyProcessor } from './EntityPolicyProcessor';
export { FileReaderProcessor } from './FileReaderProcessor';
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
export { GithubReaderProcessor } from './GithubReaderProcessor';
plugins/catalog-backend/src/ingestion/processors/types.ts
+22 -4
View File
@@ -46,15 +46,33 @@ export type CatalogProcessor = {
): Promise<boolean>;
/**
* Processes an emitted entity, e.g. by validating or modifying it.
* Pre-processes an emitted entity, after it has been emitted but before it
* has been validated.
*
* @param entity The entity to process
* This type of processing usually involves enriching the entity with
* additional data, and the input entity may actually still be incomplete
* when the processor is invoked.
*
* @param entity The (possibly partial) entity to process
* @param location The location that the entity came from
* @param read Reads the contents of a location
* @param emit A sink for auxiliary items resulting from the processing
* @returns The same entity or a modified version of it
*/
processEntity?(
preProcessEntity?(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity>;
/**
* Post-processes an emitted entity, after it has been validated.
*
* @param entity The entity to process
* @param location The location that the entity came from
* @param emit A sink for auxiliary items resulting from the processing
* @returns The same entity or a modified version of it
*/
postProcessEntity?(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+46 -45
View File
@@ -49,49 +49,6 @@ describe('CatalogBuilder', () => {
reader.read.mockResolvedValue(Buffer.from('junk'));
const builder = new CatalogBuilder(env)
.replaceReaderProcessors([
{
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
},
])
.replaceParserProcessors([
{
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
kind: 'Component',
metadata: { name: 'n' },
}),
);
return true;
},
},
])
.replacePreProcessors([
{
async processEntity(entity: Entity) {
expect(entity.apiVersion).toBe('av');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
},
])
.replaceEntityPolicies([
{
async enforce(entity: Entity) {
@@ -108,9 +65,51 @@ describe('CatalogBuilder', () => {
},
},
])
.replacePostProcessors([
.setPlaceholderResolver('t', async ({ value }) => {
expect(value).toBe('tt');
return 'tt2';
})
.setFieldFormatValidators({
isValidEntityName: n => {
expect(n).toBe('n');
return true;
},
})
.replaceProcessors([
{
async processEntity(entity: Entity) {
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
kind: 'Component',
metadata: { name: 'n', replaced: { $t: 'tt' } },
}),
);
return true;
},
async preProcessEntity(entity: Entity) {
expect(entity.apiVersion).toBe('av');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
async postProcessEntity(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return {
...entity,
metadata: { ...entity.metadata, post: 'p' },
@@ -124,6 +123,7 @@ describe('CatalogBuilder', () => {
type: 'test',
target: '',
});
expect.assertions(8);
expect(added.entities).toEqual([
{
apiVersion: 'av',
@@ -132,6 +132,7 @@ describe('CatalogBuilder', () => {
name: 'n',
namespace: 'ns',
post: 'p',
replaced: 'tt2',
}),
},
]);
plugins/catalog-backend/src/service/CatalogBuilder.ts
+91 -225
View File
@@ -48,7 +48,6 @@ import {
BitbucketApiReaderProcessor,
CatalogProcessor,
CodeOwnersProcessor,
EntityPolicyProcessor,
FileReaderProcessor,
GithubOrgReaderProcessor,
GithubReaderProcessor,
@@ -84,36 +83,24 @@ export type CatalogEnvironment = {
*
* The touch points where you can replace or extend behavior are as follows:
*
* - Reader processors can be added or replaced. These implement the
* functionality of reading raw data from a location, in the form of an
* entity definition file.
* - Parser processors can be added or replaced. These accept the raw data as
* read by the previous processors and parse it into raw structured data
* (for example, from a binary buffer containing yaml text, to a JS object
* structure).
* - Entity policies can be added or replaced. These are automatically run
* after the processors' pre-processing steps. All policies are given the
* chance to inspect the entity, and all of them have to pass in order for
* the entity to be considered valid from an overall point of view.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which adds
* additional kind-specific validation (usually based on a schema). Only one
* of the entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Placeholder resolvers can be replaced or added. These run on the raw
* structured data between the parsing and pre-processing steps, to replace
* dollar-prefixed entries with their actual values (like $file).
* - Pre-processors can be added or replaced. These take the raw unvalidated
* data from the parser processors and can enrich or extend it before
* validation. This is the place where for example codeowners data can be
* injected into partial entity definitions.
* - Entity policies can be added or replaced. These are the first line of
* validation from the output of the pre-processing step. All policies are
* given the chance to inspect the entity, and all of them have to pass in
* order for the entity to be considered valid from an overall point of
* view.
* - Field format validators can be replaced. These check the format of
* individual core fields such as metadata.name, such that they adhere to
* certain rules.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which add additional
* kind-specific validation (usually based on a schema). Only one of the
* entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Post-processors can be added or replaced. These take the validated
* entities out of the validation step and can perform additional actions on
* them.
* individual core fields such as metadata.name, to ensure that they adhere
* to certain rules.
* - Processors can be added or replaced. These implement the functionality of
* reading, parsing and processing the entity data before it is persisted in
* the catalog.
*/
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
@@ -121,16 +108,10 @@ export class CatalogBuilder {
private entityPoliciesReplace: boolean;
private entityKinds: EntityPolicy[];
private entityKindsReplace: boolean;
private readerProcessors: CatalogProcessor[];
private readerProcessorsReplace: boolean;
private parserProcessors: CatalogProcessor[];
private parserProcessorsReplace: boolean;
private preProcessors: CatalogProcessor[];
private preProcessorsReplace: boolean;
private postProcessors: CatalogProcessor[];
private postProcessorsReplace: boolean;
private placeholderResolvers: Record<string, PlaceholderResolver>;
private fieldFormatValidators: Partial<Validators>;
private processors: CatalogProcessor[];
private processorsReplace: boolean;
constructor(env: CatalogEnvironment) {
this.env = env;
@@ -138,16 +119,10 @@ export class CatalogBuilder {
this.entityPoliciesReplace = false;
this.entityKinds = [];
this.entityKindsReplace = false;
this.readerProcessors = [];
this.readerProcessorsReplace = false;
this.parserProcessors = [];
this.parserProcessorsReplace = false;
this.preProcessors = [];
this.preProcessorsReplace = false;
this.postProcessors = [];
this.postProcessorsReplace = false;
this.placeholderResolvers = {};
this.fieldFormatValidators = {};
this.processors = [];
this.processorsReplace = false;
}
/**
@@ -212,114 +187,6 @@ export class CatalogBuilder {
return this;
}
/**
* Adds processors that support reading of definition files. These are run
* before the entities are parsed, pre-processed, validated and post-
* processed.
*
* @param processors One or more processors
*/
addReaderProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.readerProcessors.push(...processors);
return this;
}
/**
* Sets what processors to use for the reading of definition files. These are
* run before the entities are parsed, pre-processed, validated and post-
* processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceReaderProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.readerProcessors = [...processors];
this.readerProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* @param processors One or more processors
*/
addParserProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.parserProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceParserProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.parserProcessors = [...processors];
this.parserProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* @param processors One or more processors
*/
addPreProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.preProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePreProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.preProcessors = [...processors];
this.preProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* @param processors One or more processors
*/
addPostProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.postProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePostProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.postProcessors = [...processors];
this.postProcessorsReplace = true;
return this;
}
/**
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
* definition files.
@@ -327,8 +194,12 @@ export class CatalogBuilder {
* @param key The key that identifies the placeholder, e.g. "file"
* @param resolver The resolver that gets values for this placeholder
*/
setPlaceholderResolver(key: string, resolver: PlaceholderResolver) {
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): CatalogBuilder {
this.placeholderResolvers[key] = resolver;
return this;
}
/**
@@ -341,8 +212,34 @@ export class CatalogBuilder {
*
* @param validators The (subset of) validators to set
*/
setFieldFormatValidators(validators: Partial<Validators>) {
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {
lodash.merge(this.fieldFormatValidators, validators);
return this;
}
/**
* Adds entity processors. These are responsible for reading, parsing, and
* processing entities before they are persisted in the catalog.
*
* @param processors One or more processors
*/
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.processors.push(...processors);
return this;
}
/**
* Sets what entity processors to use. These are responsible for reading,
* parsing, and processing entities before they are persisted in the catalog.
*
* This function replaces the default set of processors; use with care.
*
* @param processors One or more processors
*/
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.processors = [...processors];
this.processorsReplace = true;
return this;
}
/**
@@ -355,14 +252,15 @@ export class CatalogBuilder {
}> {
const { config, database, logger } = this.env;
const entityPolicy = this.buildEntityPolicy();
const processors = this.buildProcessors(entityPolicy);
const policy = this.buildEntityPolicy();
const processors = this.buildProcessors();
const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config);
const locationReader = new LocationReaders({
...this.env,
processors,
rulesEnforcer,
policy,
});
const db = await DatabaseManager.createDatabase(
@@ -418,106 +316,74 @@ export class CatalogBuilder {
]);
}
private buildProcessors(entityPolicy: EntityPolicy): CatalogProcessor[] {
const { config, reader } = this.env;
private buildProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const placeholderResolvers = lodash.merge(
{
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
},
this.placeholderResolvers,
);
const placeholderResolvers: Record<string, PlaceholderResolver> = {
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
...this.placeholderResolvers,
};
const processors = this.processorsReplace
? this.processors
: [
new FileReaderProcessor(),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
new YamlProcessor(),
new CodeOwnersProcessor({ reader }),
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
];
return [
StaticLocationProcessor.fromConfig(config),
...this.buildReaderProcessors(),
...this.buildParserProcessors(),
new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }),
...this.buildPreProcessors(),
new EntityPolicyProcessor(entityPolicy),
...this.buildPostProcessors(),
...this.buildDeprecatedReaderProcessors(),
...processors,
];
}
private buildReaderProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to
// the new integration config driven UrlReaders. In an upcoming release we
// can then completely remove support for the old processors, but still
// keep handling the deprecated location types for a while, but with a
// warning.
private buildDeprecatedReaderProcessors(): CatalogProcessor[] {
const { config, logger } = this.env;
if (this.readerProcessorsReplace) {
return this.readerProcessors;
}
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to the new
// integration config driven UrlReaders. In an upcoming release we can then completely
// remove support for the old processors, but still keep handling the deprecated location
// types for a while, but with a warning.
const oldProcessors = [];
const result = [];
const pc = config.getOptionalConfig('catalog.processors');
if (pc?.has('github')) {
logger.warn(
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
);
oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger));
result.push(GithubReaderProcessor.fromConfig(config, logger));
}
if (pc?.has('gitlabApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
);
oldProcessors.push(new GitlabApiReaderProcessor(config));
oldProcessors.push(new GitlabReaderProcessor());
result.push(new GitlabApiReaderProcessor(config));
result.push(new GitlabReaderProcessor());
}
if (pc?.has('bitbucketApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
);
oldProcessors.push(new BitbucketApiReaderProcessor(config));
result.push(new BitbucketApiReaderProcessor(config));
}
if (pc?.has('azureApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
);
oldProcessors.push(new AzureApiReaderProcessor(config));
result.push(new AzureApiReaderProcessor(config));
}
return [
new FileReaderProcessor(),
...oldProcessors,
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
...this.readerProcessors,
];
}
private buildParserProcessors(): CatalogProcessor[] {
if (this.parserProcessorsReplace) {
return this.parserProcessors;
}
return [new YamlProcessor(), ...this.parserProcessors];
}
private buildPreProcessors(): CatalogProcessor[] {
const { reader } = this.env;
if (this.preProcessorsReplace) {
return this.preProcessors;
}
return [new CodeOwnersProcessor({ reader }), ...this.preProcessors];
}
private buildPostProcessors(): CatalogProcessor[] {
if (this.postProcessorsReplace) {
return this.postProcessors;
}
return [
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
...this.postProcessors,
];
return result;
}
}