paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 44 Packages Projects Releases Wiki Activity

Merge pull request #31259 from backstage/freben/proc-prov-breakout

Browse Source 
break out provider/processor handling into a separate file
This commit is contained in:
Fredrik Adelöw
2025-09-24 14:41:39 +02:00
committed by GitHub
parent f2c2507f6a e037246a3a
commit 80f7b646a5
3 changed files with 282 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/service/CatalogBuilder.ts
+4 -47
View File
@@ -111,6 +111,7 @@ import {
catalogEntityPermissionResourceRef,
CatalogPermissionRuleInput,
} from '@backstage/plugin-catalog-node/alpha';
import { filterAndSortProcessors, filterProviders } from './util';
export type CatalogEnvironment = {
logger: LoggerService;
@@ -525,11 +526,12 @@ export class CatalogBuilder {
const locationStore = new DefaultLocationStore(dbClient);
const configLocationProvider = new ConfigLocationEntityProvider(config);
const entityProviders = this.filterProviders(
const entityProviders = filterProviders(
lodash.uniqBy(
[...this.entityProviders, locationStore, configLocationProvider],
provider => provider.getProviderName(),
),
config,
);
const processingEngine = new DefaultCatalogProcessingEngine({
@@ -684,42 +686,7 @@ export class CatalogBuilder {
this.checkMissingExternalProcessors(processors);
const filteredProcessors = this.filterProcessors(processors);
// Lastly sort the processors by priority. Config can override the
// priority of a processor to allow control of 3rd party processors.
filteredProcessors.sort((a, b) => {
const getProcessorPriority = (processor: CatalogProcessor) => {
try {
return (
config.getOptionalNumber(
`catalog.processorOptions.${processor.getProcessorName()}.priority`,
) ??
processor.getPriority?.() ??
20
);
} catch (_) {
// In case the processor config throws, just return default priority
return 20;
}
};
const aPriority = getProcessorPriority(a);
const bPriority = getProcessorPriority(b);
return aPriority - bPriority;
});
return filteredProcessors;
}
private filterProcessors(processors: CatalogProcessor[]) {
const { config } = this.env;
return processors.filter(
p =>
config.getOptionalBoolean(
`catalog.processorOptions.${p.getProcessorName()}.disabled`,
) !== true,
);
return filterAndSortProcessors(processors, config);
}
// TODO(Rugvip): These old processors are removed, for a while we'll be throwing
@@ -832,16 +799,6 @@ export class CatalogBuilder {
);
}
private filterProviders(providers: EntityProvider[]) {
const { config } = this.env;
return providers.filter(
p =>
config.getOptionalBoolean(
`catalog.providerOptions.${p.getProviderName()}.disabled`,
) !== true,
);
}
private static getDefaultProcessingInterval(
config: Config,
): ProcessingIntervalFunction {
plugins/catalog-backend/src/service/util.test.ts
+195
View File
@@ -0,0 +1,195 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { mockServices } from '@backstage/backend-test-utils';
import { filterAndSortProcessors, filterProviders } from './util';
describe('filterAndSortProcessors', () => {
it('should filter processors', () => {
const p1 = { getProcessorName: () => 'processor1' };
const p2 = { getProcessorName: () => 'processor2' };
expect(
filterAndSortProcessors([p1, p2], mockServices.rootConfig({ data: {} })),
).toEqual([p1, p2]);
expect(
filterAndSortProcessors(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
processorOptions: {
processor1: { disabled: true },
},
},
},
}),
),
).toEqual([p2]);
expect(
filterAndSortProcessors(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
processorOptions: {
processor2: { disabled: true },
},
},
},
}),
),
).toEqual([p1]);
});
it('should sort processors', () => {
const p1 = { getProcessorName: () => 'processor1', getPriority: () => 10 };
const p2 = { getProcessorName: () => 'processor2' };
const p3 = {
getProcessorName: () => 'processor3',
getPriority: () => {
throw new Error('failed');
},
};
expect(
filterAndSortProcessors(
[p1, p2, p3],
mockServices.rootConfig({ data: {} }),
),
).toEqual([p1, p2, p3]); // p2 and p3 got the defaults
expect(
filterAndSortProcessors(
[p1, p2, p3],
mockServices.rootConfig({
data: {
catalog: {
processorOptions: {
processor2: { priority: 2 },
processor3: { priority: 1 },
},
},
},
}),
),
).toEqual([p3, p2, p1]);
});
it('rejects invalid config', () => {
const p1 = { getProcessorName: () => 'processor1' };
const p2 = { getProcessorName: () => 'processor1' };
expect(() =>
filterAndSortProcessors(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
processorOptions: {
processor1: { disabled: 'i guess so, maybe' },
},
},
},
}),
),
).toThrowErrorMatchingInlineSnapshot(
`"Unable to convert config value for key 'catalog.processorOptions.processor1.disabled' in 'mock-config' to a boolean"`,
);
expect(() =>
filterAndSortProcessors(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
processorOptions: {
processor1: { priority: 'somewhere in the middle, roughly' },
},
},
},
}),
),
).toThrowErrorMatchingInlineSnapshot(
`"Unable to convert config value for key 'catalog.processorOptions.processor1.priority' in 'mock-config' to a number"`,
);
});
});
describe('filterProviders', () => {
it('should filter providers', () => {
const p1 = { getProviderName: () => 'provider1', connect: jest.fn() };
const p2 = { getProviderName: () => 'provider2', connect: jest.fn() };
expect(
filterProviders([p1, p2], mockServices.rootConfig({ data: {} })),
).toEqual([p1, p2]);
expect(
filterProviders(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
providerOptions: {
provider1: { disabled: true },
},
},
},
}),
),
).toEqual([p2]);
expect(
filterProviders(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
providerOptions: {
provider2: { disabled: true },
},
},
},
}),
),
).toEqual([p1]);
});
it('rejects invalid config', () => {
const p1 = { getProviderName: () => 'provider1', connect: jest.fn() };
const p2 = { getProviderName: () => 'provider2', connect: jest.fn() };
expect(() =>
filterProviders(
[p1, p2],
mockServices.rootConfig({
data: {
catalog: {
providerOptions: {
provider1: { disabled: 'i guess so, maybe' },
},
},
},
}),
),
).toThrowErrorMatchingInlineSnapshot(
`"Unable to convert config value for key 'catalog.providerOptions.provider1.disabled' in 'mock-config' to a boolean"`,
);
});
});
plugins/catalog-backend/src/service/util.ts
+83 -1
View File
@@ -24,12 +24,17 @@ import {
QueryEntitiesInitialRequest,
QueryEntitiesRequest,
} from '../catalog/types';
import { EntityFilter } from '@backstage/plugin-catalog-node';
import {
CatalogProcessor,
EntityFilter,
EntityProvider,
} from '@backstage/plugin-catalog-node';
import {
Entity,
parseEntityRef,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
export async function requireRequestBody(req: Request): Promise<unknown> {
const contentType = req.header('content-type');
@@ -166,3 +171,80 @@ export function expandLegacyCompoundRelationsInEntity(entity: Entity): Entity {
}
return entity;
}
/**
* Given a list of catalog processors, filter out the ones that are disabled
* through the `catalog.processorOptions` config and sort them by priority.
*/
export function filterAndSortProcessors(
processors: CatalogProcessor[],
config: Config,
): CatalogProcessor[] {
function getProcessorOptions(
processor: CatalogProcessor,
): Config | undefined {
const root = config.getOptionalConfig('catalog.processorOptions');
try {
return root?.getOptionalConfig(processor.getProcessorName());
} catch {
// We silence errors specifically here, to cover for cases where the
// processor name contains special characters which makes the config
// reader throw an error.
return undefined;
}
}
function isProcessorDisabled(processor: CatalogProcessor): boolean {
return (
getProcessorOptions(processor)?.getOptionalBoolean('disabled') === true
);
}
function getProcessorPriority(processor: CatalogProcessor): number {
let priority =
getProcessorOptions(processor)?.getOptionalNumber('priority');
if (priority === undefined) {
try {
priority = processor.getPriority?.();
} catch {
// In case the processor method throws, just return default priority
}
}
return priority ?? 20;
}
return processors
.filter(p => !isProcessorDisabled(p))
.sort((a, b) => getProcessorPriority(a) - getProcessorPriority(b));
}
/**
* Given a list of entity providers, filter out the ones that are disabled
* through the `catalog.providerOptions` config.
*/
export function filterProviders(
providers: EntityProvider[],
config: Config,
): EntityProvider[] {
function getProviderOptions(provider: EntityProvider): Config | undefined {
const root = config.getOptionalConfig('catalog.providerOptions');
try {
return root?.getOptionalConfig(provider.getProviderName());
} catch {
// We silence errors specifically here, to cover for cases where the
// provider name contains special characters which makes the config
// reader throw an error.
return undefined;
}
}
function isProviderDisabled(provider: EntityProvider): boolean {
return (
getProviderOptions(provider)?.getOptionalBoolean('disabled') === true
);
}
return providers.filter(p => !isProviderDisabled(p));
}