paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

Merge pull request #2955 from spotify/freben/next-steps-in-processing

Browse Source 
feat(catalog-backend): First steps of refactoring - split processEntity
This commit is contained in:
Fredrik Adelöw
2020-10-20 14:13:49 +02:00
committed by GitHub
parent b611ea380f 4036ff59d0
commit 17e04965b0
13 changed files with 257 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+58 -14
View File
@@ -17,6 +17,7 @@
import { UrlReader } from '@backstage/backend-common';
import {
Entity,
EntityPolicy,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
} from '@backstage/catalog-model';
@@ -44,6 +45,7 @@ type Options = {
config: Config;
processors: CatalogProcessor[];
rulesEnforcer: CatalogRulesEnforcer;
policy: EntityPolicy;
};
/**
@@ -74,10 +76,12 @@ export class LocationReaders implements LocationReader {
} else if (item.type === 'entity') {
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
const entity = await this.handleEntity(item, emit);
output.entities.push({
entity,
location: item.location,
});
if (entity) {
output.entities.push({
entity,
location: item.location,
});
}
} else {
output.errors.push({
location: item.location,
@@ -162,25 +166,65 @@ export class LocationReaders implements LocationReader {
private async handleEntity(
item: CatalogProcessorEntityResult,
emit: CatalogProcessorEmit,
): Promise<Entity> {
): Promise<Entity | undefined> {
const { processors, logger } = this.options;
let current = item.entity;
// Construct the name carefully, this happens before validation below
// so we do not want to crash here due to missing metadata or so
const kind = current.kind || '';
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
for (const processor of processors) {
if (processor.processEntity) {
if (processor.preProcessEntity) {
try {
current = await processor.processEntity(current, item.location, emit);
current = await processor.preProcessEntity(
current,
item.location,
emit,
);
} catch (e) {
// Construct the name carefully, if we got validation errors we do
// not want to crash here due to missing metadata or so
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
const message = `Processor ${processor.constructor.name} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
try {
const next = await this.options.policy.enforce(current);
if (!next) {
const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
current = next;
} catch (e) {
const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.inputError(item.location, message));
logger.warn(message);
return undefined;
}
for (const processor of processors) {
if (processor.postProcessEntity) {
try {
current = await processor.postProcessEntity(
current,
item.location,
emit,
);
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
return undefined;
}
}
}
plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
+4 -1
View File
@@ -19,7 +19,10 @@ import lodash from 'lodash';
import { CatalogProcessor } from './types';
export class AnnotateLocationEntityProcessor implements CatalogProcessor {
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
return lodash.merge(
{
metadata: {
plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
+4 -4
View File
@@ -224,7 +224,7 @@ describe('CodeOwnersProcessor', () => {
spec: { owner: '@acme/foo-team' },
});
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
@@ -235,7 +235,7 @@ describe('CodeOwnersProcessor', () => {
it('should ignore url locations', async () => {
const { entity, processor } = setupTest();
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation({ type: 'url' }),
);
@@ -246,7 +246,7 @@ describe('CodeOwnersProcessor', () => {
it('should ignore invalid kinds', async () => {
const { entity, processor } = setupTest({ kind: 'Group' });
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
@@ -257,7 +257,7 @@ describe('CodeOwnersProcessor', () => {
it('should set owner from codeowner', async () => {
const { entity, processor } = setupTest();
const result = await processor.processEntity(
const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
);
plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts
+4 -1
View File
@@ -40,7 +40,10 @@ type Options = {
export class CodeOwnersProcessor implements CatalogProcessor {
constructor(private readonly options: Options) {}
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
// Only continue if the owner is not set
if (
!entity ||
plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts
-34
View File
@@ -1,34 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import { CatalogProcessor } from './types';
export class EntityPolicyProcessor implements CatalogProcessor {
private readonly policy: EntityPolicy;
constructor(policy: EntityPolicy) {
this.policy = policy;
}
async processEntity(entity: Entity): Promise<Entity> {
const output = await this.policy.enforce(entity);
if (!output) {
throw new Error(`Entity did not match any known schema`);
}
return output;
}
}
plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts
+1 -1
View File
@@ -19,7 +19,7 @@ import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
export class LocationRefProcessor implements CatalogProcessor {
async processEntity(
async postProcessEntity(
entity: Entity,
_location: LocationSpec,
emit: CatalogProcessorEmit,
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts
+10 -10
View File
@@ -46,7 +46,7 @@ describe('PlaceholderProcessor', () => {
reader,
});
await expect(
processor.processEntity(input, { type: 't', target: 'l' }),
processor.preProcessEntity(input, { type: 't', target: 'l' }),
).resolves.toBe(input);
});
@@ -62,7 +62,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -98,7 +98,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -122,7 +122,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -143,7 +143,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -177,7 +177,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -209,7 +209,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -241,7 +241,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -277,7 +277,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
@@ -316,7 +316,7 @@ describe('PlaceholderProcessor', () => {
});
await expect(
processor.processEntity(
processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts
+4 -1
View File
@@ -45,7 +45,10 @@ type Options = {
export class PlaceholderProcessor implements CatalogProcessor {
constructor(private readonly options: Options) {}
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
): Promise<Entity> {
const process = async (data: any): Promise<[any, boolean]> => {
if (!data || !(data instanceof Object)) {
// Scalars can't have placeholders
plugins/catalog-backend/src/ingestion/processors/index.ts
-1
View File
@@ -23,7 +23,6 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess
export { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
export { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
export { CodeOwnersProcessor } from './CodeOwnersProcessor';
export { EntityPolicyProcessor } from './EntityPolicyProcessor';
export { FileReaderProcessor } from './FileReaderProcessor';
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
export { GithubReaderProcessor } from './GithubReaderProcessor';
plugins/catalog-backend/src/ingestion/processors/types.ts
+22 -4
View File
@@ -46,15 +46,33 @@ export type CatalogProcessor = {
): Promise<boolean>;
/**
* Processes an emitted entity, e.g. by validating or modifying it.
* Pre-processes an emitted entity, after it has been emitted but before it
* has been validated.
*
* @param entity The entity to process
* This type of processing usually involves enriching the entity with
* additional data, and the input entity may actually still be incomplete
* when the processor is invoked.
*
* @param entity The (possibly partial) entity to process
* @param location The location that the entity came from
* @param read Reads the contents of a location
* @param emit A sink for auxiliary items resulting from the processing
* @returns The same entity or a modified version of it
*/
processEntity?(
preProcessEntity?(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity>;
/**
* Post-processes an emitted entity, after it has been validated.
*
* @param entity The entity to process
* @param location The location that the entity came from
* @param emit A sink for auxiliary items resulting from the processing
* @returns The same entity or a modified version of it
*/
postProcessEntity?(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+46 -45
View File
@@ -49,49 +49,6 @@ describe('CatalogBuilder', () => {
reader.read.mockResolvedValue(Buffer.from('junk'));
const builder = new CatalogBuilder(env)
.replaceReaderProcessors([
{
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
},
])
.replaceParserProcessors([
{
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
kind: 'Component',
metadata: { name: 'n' },
}),
);
return true;
},
},
])
.replacePreProcessors([
{
async processEntity(entity: Entity) {
expect(entity.apiVersion).toBe('av');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
},
])
.replaceEntityPolicies([
{
async enforce(entity: Entity) {
@@ -108,9 +65,51 @@ describe('CatalogBuilder', () => {
},
},
])
.replacePostProcessors([
.setPlaceholderResolver('t', async ({ value }) => {
expect(value).toBe('tt');
return 'tt2';
})
.setFieldFormatValidators({
isValidEntityName: n => {
expect(n).toBe('n');
return true;
},
})
.replaceProcessors([
{
async processEntity(entity: Entity) {
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
kind: 'Component',
metadata: { name: 'n', replaced: { $t: 'tt' } },
}),
);
return true;
},
async preProcessEntity(entity: Entity) {
expect(entity.apiVersion).toBe('av');
return {
...entity,
metadata: { ...entity.metadata, namespace: 'ns' },
};
},
async postProcessEntity(entity: Entity) {
expect(entity.metadata.namespace).toBe('ns');
return {
...entity,
metadata: { ...entity.metadata, post: 'p' },
@@ -124,6 +123,7 @@ describe('CatalogBuilder', () => {
type: 'test',
target: '',
});
expect.assertions(8);
expect(added.entities).toEqual([
{
apiVersion: 'av',
@@ -132,6 +132,7 @@ describe('CatalogBuilder', () => {
name: 'n',
namespace: 'ns',
post: 'p',
replaced: 'tt2',
}),
},
]);
plugins/catalog-backend/src/service/CatalogBuilder.ts
+91 -225
View File
@@ -47,7 +47,6 @@ import {
BitbucketApiReaderProcessor,
CatalogProcessor,
CodeOwnersProcessor,
EntityPolicyProcessor,
FileReaderProcessor,
GithubOrgReaderProcessor,
GithubReaderProcessor,
@@ -83,36 +82,24 @@ export type CatalogEnvironment = {
*
* The touch points where you can replace or extend behavior are as follows:
*
* - Reader processors can be added or replaced. These implement the
* functionality of reading raw data from a location, in the form of an
* entity definition file.
* - Parser processors can be added or replaced. These accept the raw data as
* read by the previous processors and parse it into raw structured data
* (for example, from a binary buffer containing yaml text, to a JS object
* structure).
* - Entity policies can be added or replaced. These are automatically run
* after the processors' pre-processing steps. All policies are given the
* chance to inspect the entity, and all of them have to pass in order for
* the entity to be considered valid from an overall point of view.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which adds
* additional kind-specific validation (usually based on a schema). Only one
* of the entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Placeholder resolvers can be replaced or added. These run on the raw
* structured data between the parsing and pre-processing steps, to replace
* dollar-prefixed entries with their actual values (like $file).
* - Pre-processors can be added or replaced. These take the raw unvalidated
* data from the parser processors and can enrich or extend it before
* validation. This is the place where for example codeowners data can be
* injected into partial entity definitions.
* - Entity policies can be added or replaced. These are the first line of
* validation from the output of the pre-processing step. All policies are
* given the chance to inspect the entity, and all of them have to pass in
* order for the entity to be considered valid from an overall point of
* view.
* - Field format validators can be replaced. These check the format of
* individual core fields such as metadata.name, such that they adhere to
* certain rules.
* - Entity kinds can be added or replaced. These are the second line of
* validation that is applied after the entity policies, which add additional
* kind-specific validation (usually based on a schema). Only one of the
* entity kinds has to accept the entity, but if none of them do, the
* entity is rejected as a whole.
* - Post-processors can be added or replaced. These take the validated
* entities out of the validation step and can perform additional actions on
* them.
* individual core fields such as metadata.name, to ensure that they adhere
* to certain rules.
* - Processors can be added or replaced. These implement the functionality of
* reading, parsing and processing the entity data before it is persisted in
* the catalog.
*/
export class CatalogBuilder {
private readonly env: CatalogEnvironment;
@@ -120,16 +107,10 @@ export class CatalogBuilder {
private entityPoliciesReplace: boolean;
private entityKinds: EntityPolicy[];
private entityKindsReplace: boolean;
private readerProcessors: CatalogProcessor[];
private readerProcessorsReplace: boolean;
private parserProcessors: CatalogProcessor[];
private parserProcessorsReplace: boolean;
private preProcessors: CatalogProcessor[];
private preProcessorsReplace: boolean;
private postProcessors: CatalogProcessor[];
private postProcessorsReplace: boolean;
private placeholderResolvers: Record<string, PlaceholderResolver>;
private fieldFormatValidators: Partial<Validators>;
private processors: CatalogProcessor[];
private processorsReplace: boolean;
constructor(env: CatalogEnvironment) {
this.env = env;
@@ -137,16 +118,10 @@ export class CatalogBuilder {
this.entityPoliciesReplace = false;
this.entityKinds = [];
this.entityKindsReplace = false;
this.readerProcessors = [];
this.readerProcessorsReplace = false;
this.parserProcessors = [];
this.parserProcessorsReplace = false;
this.preProcessors = [];
this.preProcessorsReplace = false;
this.postProcessors = [];
this.postProcessorsReplace = false;
this.placeholderResolvers = {};
this.fieldFormatValidators = {};
this.processors = [];
this.processorsReplace = false;
}
/**
@@ -211,114 +186,6 @@ export class CatalogBuilder {
return this;
}
/**
* Adds processors that support reading of definition files. These are run
* before the entities are parsed, pre-processed, validated and post-
* processed.
*
* @param processors One or more processors
*/
addReaderProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.readerProcessors.push(...processors);
return this;
}
/**
* Sets what processors to use for the reading of definition files. These are
* run before the entities are parsed, pre-processed, validated and post-
* processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceReaderProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.readerProcessors = [...processors];
this.readerProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* @param processors One or more processors
*/
addParserProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.parserProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each definition file has been read, in
* order to parse the raw data. These are run before the entities are
* pre-processed, validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replaceParserProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.parserProcessors = [...processors];
this.parserProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* @param processors One or more processors
*/
addPreProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.preProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read and parsed,
* but before being validated and post-processed.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePreProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.preProcessors = [...processors];
this.preProcessorsReplace = true;
return this;
}
/**
* Adds processors that run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* @param processors One or more processors
*/
addPostProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.postProcessors.push(...processors);
return this;
}
/**
* Sets what processors to run after each entity has been read, parsed,
* run through the pre-processors, and validated.
*
* This function replaces the default set of processors in this stage; use
* with care.
*
* @param processors One or more processors
*/
replacePostProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.postProcessors = [...processors];
this.postProcessorsReplace = true;
return this;
}
/**
* Adds, or overwrites, a handler for placeholders (e.g. $file) in entity
* definition files.
@@ -326,8 +193,12 @@ export class CatalogBuilder {
* @param key The key that identifies the placeholder, e.g. "file"
* @param resolver The resolver that gets values for this placeholder
*/
setPlaceholderResolver(key: string, resolver: PlaceholderResolver) {
setPlaceholderResolver(
key: string,
resolver: PlaceholderResolver,
): CatalogBuilder {
this.placeholderResolvers[key] = resolver;
return this;
}
/**
@@ -340,8 +211,34 @@ export class CatalogBuilder {
*
* @param validators The (subset of) validators to set
*/
setFieldFormatValidators(validators: Partial<Validators>) {
setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {
lodash.merge(this.fieldFormatValidators, validators);
return this;
}
/**
* Adds entity processors. These are responsible for reading, parsing, and
* processing entities before they are persisted in the catalog.
*
* @param processors One or more processors
*/
addProcessor(...processors: CatalogProcessor[]): CatalogBuilder {
this.processors.push(...processors);
return this;
}
/**
* Sets what entity processors to use. These are responsible for reading,
* parsing, and processing entities before they are persisted in the catalog.
*
* This function replaces the default set of processors; use with care.
*
* @param processors One or more processors
*/
replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {
this.processors = [...processors];
this.processorsReplace = true;
return this;
}
/**
@@ -354,14 +251,15 @@ export class CatalogBuilder {
}> {
const { config, database, logger } = this.env;
const entityPolicy = this.buildEntityPolicy();
const processors = this.buildProcessors(entityPolicy);
const policy = this.buildEntityPolicy();
const processors = this.buildProcessors();
const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config);
const locationReader = new LocationReaders({
...this.env,
processors,
rulesEnforcer,
policy,
});
const db = await DatabaseManager.createDatabase(
@@ -416,106 +314,74 @@ export class CatalogBuilder {
]);
}
private buildProcessors(entityPolicy: EntityPolicy): CatalogProcessor[] {
const { config, reader } = this.env;
private buildProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
const placeholderResolvers = lodash.merge(
{
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
},
this.placeholderResolvers,
);
const placeholderResolvers: Record<string, PlaceholderResolver> = {
json: jsonPlaceholderResolver,
yaml: yamlPlaceholderResolver,
text: textPlaceholderResolver,
...this.placeholderResolvers,
};
const processors = this.processorsReplace
? this.processors
: [
new FileReaderProcessor(),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
new YamlProcessor(),
new CodeOwnersProcessor({ reader }),
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
];
return [
StaticLocationProcessor.fromConfig(config),
...this.buildReaderProcessors(),
...this.buildParserProcessors(),
new PlaceholderProcessor({ resolvers: placeholderResolvers, reader }),
...this.buildPreProcessors(),
new EntityPolicyProcessor(entityPolicy),
...this.buildPostProcessors(),
...this.buildDeprecatedReaderProcessors(),
...processors,
];
}
private buildReaderProcessors(): CatalogProcessor[] {
const { config, logger, reader } = this.env;
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to
// the new integration config driven UrlReaders. In an upcoming release we
// can then completely remove support for the old processors, but still
// keep handling the deprecated location types for a while, but with a
// warning.
private buildDeprecatedReaderProcessors(): CatalogProcessor[] {
const { config, logger } = this.env;
if (this.readerProcessorsReplace) {
return this.readerProcessors;
}
// TODO(Rugvip): These are added for backwards compatibility if config exists
// The idea is to have everyone migrate from using the old processors to the new
// integration config driven UrlReaders. In an upcoming release we can then completely
// remove support for the old processors, but still keep handling the deprecated location
// types for a while, but with a warning.
const oldProcessors = [];
const result = [];
const pc = config.getOptionalConfig('catalog.processors');
if (pc?.has('github')) {
logger.warn(
`Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
);
oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger));
result.push(GithubReaderProcessor.fromConfig(config, logger));
}
if (pc?.has('gitlabApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
);
oldProcessors.push(new GitlabApiReaderProcessor(config));
oldProcessors.push(new GitlabReaderProcessor());
result.push(new GitlabApiReaderProcessor(config));
result.push(new GitlabReaderProcessor());
}
if (pc?.has('bitbucketApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
);
oldProcessors.push(new BitbucketApiReaderProcessor(config));
result.push(new BitbucketApiReaderProcessor(config));
}
if (pc?.has('azureApi')) {
logger.warn(
`Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
);
oldProcessors.push(new AzureApiReaderProcessor(config));
result.push(new AzureApiReaderProcessor(config));
}
return [
new FileReaderProcessor(),
...oldProcessors,
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
...this.readerProcessors,
];
}
private buildParserProcessors(): CatalogProcessor[] {
if (this.parserProcessorsReplace) {
return this.parserProcessors;
}
return [new YamlProcessor(), ...this.parserProcessors];
}
private buildPreProcessors(): CatalogProcessor[] {
const { reader } = this.env;
if (this.preProcessorsReplace) {
return this.preProcessors;
}
return [new CodeOwnersProcessor({ reader }), ...this.preProcessors];
}
private buildPostProcessors(): CatalogProcessor[] {
if (this.postProcessorsReplace) {
return this.postProcessors;
}
return [
new LocationRefProcessor(),
new AnnotateLocationEntityProcessor(),
...this.postProcessors,
];
return result;
}
}