Merge pull request #19337 from backstage/rugvip/module-extensions

backend-plugin-api: allow modules to register extension points
This commit is contained in:
Patrik Oldsberg
2023-08-14 14:34:56 +02:00
committed by GitHub
15 changed files with 798 additions and 89 deletions
@@ -0,0 +1,245 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DependencyGraph } from './DependencyGraph';
describe('DependencyGraph', () => {
it('should be empty', async () => {
const empty = DependencyGraph.fromMap({});
expect(empty.findUnsatisfiedDeps()).toEqual([]);
expect(empty.detectCircularDependency()).toBeUndefined();
await expect(
empty.parallelTopologicalTraversal(async id => id),
).resolves.toEqual([]);
});
describe('detectCircularDependency', () => {
it('should return undefined with not deps', () => {
expect(
DependencyGraph.fromMap({
1: {},
2: {},
3: {},
4: {},
}).detectCircularDependency(),
).toBeUndefined();
});
it('should return undefined with no circular deps', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { consumes: ['a'], provides: ['b', 'c'] },
3: { consumes: ['b'] },
4: { consumes: ['c'] },
}).detectCircularDependency(),
).toBeUndefined();
});
it('should detect an immediate circular dep', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'], consumes: ['a'] },
}).detectCircularDependency(),
).toEqual(['1', '1']);
});
it('should detect a small circular dep', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'], consumes: ['b'] },
2: { provides: ['b'], consumes: ['a'] },
}).detectCircularDependency(),
).toEqual(['1', '2', '1']);
});
it('should detect a larger distant circular dep', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { provides: ['b'], consumes: ['a', 'e'] },
3: { provides: ['c'], consumes: ['b'] },
4: { provides: ['d', 'e'], consumes: ['c', 'a'] },
}).detectCircularDependency(),
).toEqual(['2', '3', '4', '2']);
});
});
describe('findUnsatisfiedDeps', () => {
it('should return nothing with no deps', () => {
expect(
DependencyGraph.fromMap({
1: {},
2: {},
3: {},
4: {},
}).findUnsatisfiedDeps(),
).toEqual([]);
});
it('should return nothing when all deps are satisfied', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { consumes: ['a'], provides: ['b', 'c'] },
3: { consumes: ['b'] },
4: { consumes: ['c'] },
}).findUnsatisfiedDeps(),
).toEqual([]);
});
it('should find a single unsatisfied dep', async () => {
expect(
DependencyGraph.fromMap({
1: { consumes: ['a'] },
}).findUnsatisfiedDeps(),
).toEqual([{ value: '1', unsatisfied: ['a'] }]);
});
it('should handle circular dependencies', async () => {
expect(
DependencyGraph.fromMap({
1: { consumes: ['a'], provides: ['a'] },
}).findUnsatisfiedDeps(),
).toEqual([]);
expect(
DependencyGraph.fromMap({
1: { consumes: ['a'], provides: ['b'] },
2: { consumes: ['b'], provides: ['a'] },
}).findUnsatisfiedDeps(),
).toEqual([]);
expect(
DependencyGraph.fromMap({
1: { consumes: ['a'] },
2: { consumes: ['b'], provides: ['c'] },
3: { consumes: ['c'], provides: ['a', 'b'] },
}).findUnsatisfiedDeps(),
).toEqual([]);
});
it('should find multiple unsatisfied deps for one node', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'], consumes: ['b'] },
2: { provides: ['b'], consumes: ['a', 'd', 'e'] },
}).findUnsatisfiedDeps(),
).toEqual([{ value: '2', unsatisfied: ['d', 'e'] }]);
});
it('should find multiple unsatisfied deps for multiple nodes', async () => {
expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { provides: ['b'], consumes: ['a', 'd', 'e'] },
3: { provides: [], consumes: ['b'] },
4: { provides: [], consumes: ['c', 'a'] },
}).findUnsatisfiedDeps(),
).toEqual([
{ value: '2', unsatisfied: ['d', 'e'] },
{ value: '4', unsatisfied: ['c'] },
]);
});
});
describe('parallelTopologicalTraversal', () => {
it('should traverse with no deps', async () => {
await expect(
DependencyGraph.fromMap({
1: {},
2: {},
3: {},
4: {},
}).parallelTopologicalTraversal(async id => id),
).resolves.toEqual(['1', '2', '3', '4']);
});
it('should traverse with a few deps', async () => {
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { consumes: ['a'], provides: ['b', 'c'] },
3: { consumes: ['b'] },
4: { consumes: ['c'] },
}).parallelTopologicalTraversal(async id => id),
).resolves.toEqual(['1', '2', '3', '4']);
});
it('should traverse in reverse', async () => {
await expect(
DependencyGraph.fromMap({
1: { consumes: ['c'] },
2: { provides: ['c'], consumes: ['b'] },
3: { provides: ['b'], consumes: ['a'] },
4: { provides: ['a'] },
}).parallelTopologicalTraversal(async id => id),
).resolves.toEqual(['4', '3', '2', '1']);
});
it('should execute in parallel', async () => {
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { provides: ['b'], consumes: ['a'] },
3: { provides: ['c'], consumes: ['a'] },
4: { consumes: ['b'] },
5: { consumes: ['c'] },
}).parallelTopologicalTraversal(async id => id),
).resolves.toEqual(['1', '2', '3', '4', '5']);
// Same as above, but with 2 being delayed
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { provides: ['b'], consumes: ['a'] },
3: { provides: ['c'], consumes: ['a'] },
4: { consumes: ['b'] },
5: { consumes: ['c'] },
}).parallelTopologicalTraversal(async id => {
// When delaying 2 we expect 3 and 5 to complete before 2 and 4
if (id === '2') {
await new Promise(resolve => setTimeout(resolve, 100));
}
return id;
}),
).resolves.toEqual(['1', '3', '5', '2', '4']);
});
it('should detect circular dependencies', async () => {
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'], consumes: ['a'] },
}).parallelTopologicalTraversal(async id => id),
).rejects.toThrow('Circular dependency detected');
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'], consumes: ['b'] },
2: { provides: ['b'], consumes: ['a'] },
}).parallelTopologicalTraversal(async id => id),
).rejects.toThrow('Circular dependency detected');
await expect(
DependencyGraph.fromMap({
1: { provides: ['a'] },
2: { provides: ['c'], consumes: ['a', 'b'] },
3: { provides: ['b'], consumes: ['a', 'c'] },
}).parallelTopologicalTraversal(async id => id),
).rejects.toThrow('Circular dependency detected');
});
});
});
@@ -0,0 +1,197 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
interface NodeInput<T> {
value: T;
consumes?: Iterable<string>;
provides?: Iterable<string>;
}
/** @internal */
class Node<T> {
static from<T>(input: NodeInput<T>) {
return new Node<T>(
input.value,
input.consumes ? new Set(input.consumes) : new Set(),
input.provides ? new Set(input.provides) : new Set(),
);
}
private constructor(
readonly value: T,
readonly consumes: Set<string>,
readonly provides: Set<string>,
) {}
}
/**
* Internal helper to help validate and traverse a dependency graph.
* @internal
*/
export class DependencyGraph<T> {
static fromMap(
nodes: Record<string, Omit<NodeInput<unknown>, 'value'>>,
): DependencyGraph<string> {
return this.fromIterable(
Object.entries(nodes).map(([key, node]) => ({
value: String(key),
...node,
})),
);
}
static fromIterable<T>(
nodeInputs: Iterable<NodeInput<T>>,
): DependencyGraph<T> {
const nodes = new Array<Node<T>>();
for (const nodeInput of nodeInputs) {
nodes.push(Node.from(nodeInput));
}
return new DependencyGraph(nodes);
}
#nodes: Array<Node<T>>;
#allProvided: Set<string>;
private constructor(nodes: Array<Node<T>>) {
this.#nodes = nodes;
this.#allProvided = new Set();
for (const node of this.#nodes.values()) {
for (const produced of node.provides) {
this.#allProvided.add(produced);
}
}
}
// Find all nodes that consume dependencies that are not provided by any other node
findUnsatisfiedDeps(): Array<{ value: T; unsatisfied: string[] }> {
const unsatisfiedDependencies = [];
for (const node of this.#nodes.values()) {
const unsatisfied = Array.from(node.consumes).filter(
id => !this.#allProvided.has(id),
);
if (unsatisfied.length > 0) {
unsatisfiedDependencies.push({ value: node.value, unsatisfied });
}
}
return unsatisfiedDependencies;
}
// Detect circular dependencies within the graph, returning the path of nodes that
// form a cycle, with the same node as the first and last element of the array.
detectCircularDependency(): T[] | undefined {
for (const startNode of this.#nodes) {
const visited = new Set<Node<T>>();
const stack = new Array<[node: Node<T>, path: T[]]>([
startNode,
[startNode.value],
]);
while (stack.length > 0) {
const [node, path] = stack.pop()!;
if (visited.has(node)) {
continue;
}
visited.add(node);
for (const produced of node.provides) {
const consumerNodes = this.#nodes.filter(other =>
other.consumes.has(produced),
);
for (const consumer of consumerNodes) {
if (consumer === startNode) {
return [...path, startNode.value];
}
if (!visited.has(consumer)) {
stack.push([consumer, [...path, consumer.value]]);
}
}
}
}
}
return undefined;
}
/**
* Traverses the dependency graph in topological order, calling the provided
* function for each node and waiting for it to resolve.
*
* The nodes are traversed in parallel, but in such a way that no node is
* visited before all of its dependencies.
*
* Dependencies of nodes that are not produced by any other nodes will be ignored.
*/
async parallelTopologicalTraversal<TResult>(
fn: (value: T) => Promise<TResult>,
): Promise<TResult[]> {
const allProvided = this.#allProvided;
const producedSoFar = new Set<string>();
const waiting = new Set(this.#nodes.values());
const visited = new Set<Node<T>>();
const results = new Array<TResult>();
let inFlight = 0; // Keep track of how many callbacks are in flight, so that we know if we got stuck
// Find all nodes that have no dependencies that have not already been produced by visited nodes
async function processMoreNodes() {
if (waiting.size === 0) {
return;
}
const nodesToProcess = [];
for (const node of waiting) {
let ready = true;
for (const consumed of node.consumes) {
if (allProvided.has(consumed) && !producedSoFar.has(consumed)) {
ready = false;
continue;
}
}
if (ready) {
nodesToProcess.push(node);
}
}
for (const node of nodesToProcess) {
waiting.delete(node);
}
if (nodesToProcess.length === 0 && inFlight === 0) {
// We expect the caller to check for circular dependencies before
// traversal, so this error should never happen
throw new Error('Circular dependency detected');
}
await Promise.all(nodesToProcess.map(processNode));
}
// Process an individual node, and then add its produced dependencies to the set of available products
async function processNode(node: Node<T>) {
visited.add(node);
inFlight += 1;
const result = await fn(node.value);
results.push(result);
node.provides.forEach(produced => producedSoFar.add(produced));
inFlight -= 1;
await processMoreNodes();
}
await processMoreNodes();
return results;
}
}
@@ -20,10 +20,15 @@ import {
coreServices,
createBackendPlugin,
createBackendModule,
createExtensionPoint,
} from '@backstage/backend-plugin-api';
import { BackendInitializer } from './BackendInitializer';
import { ServiceRegistry } from './ServiceRegistry';
import { rootLifecycleServiceFactory } from '../services/implementations';
import {
lifecycleServiceFactory,
loggerServiceFactory,
rootLifecycleServiceFactory,
} from '../services/implementations';
const rootRef = createServiceRef<{ x: number }>({
id: '1',
@@ -44,6 +49,17 @@ class MockLogger {
}
}
const baseFactories = [
lifecycleServiceFactory(),
rootLifecycleServiceFactory(),
createServiceFactory({
service: coreServices.rootLogger,
deps: {},
factory: () => new MockLogger(),
})(),
loggerServiceFactory(),
];
describe('BackendInitializer', () => {
it('should initialize root scoped services', async () => {
const rootFactory = jest.fn();
@@ -75,6 +91,65 @@ describe('BackendInitializer', () => {
expect(pluginFactory).not.toHaveBeenCalled();
});
it('should initialize modules with extension points', async () => {
expect.assertions(3);
const extensionPoint = createExtensionPoint<{ values: string[] }>({
id: 'a',
});
const init = new BackendInitializer(new ServiceRegistry(baseFactories));
init.add(
createBackendModule({
pluginId: 'test',
moduleId: 'modA',
register(reg) {
reg.registerInit({
deps: { extension: extensionPoint },
async init({ extension }) {
expect(extension.values).toEqual(['b']);
extension.values.push('a');
},
});
},
})(),
);
init.add(
createBackendModule({
pluginId: 'test',
moduleId: 'modB',
register(reg) {
const values = ['b'];
reg.registerExtensionPoint(extensionPoint, { values });
reg.registerInit({
deps: {},
async init() {
expect(values).toEqual(['b', 'a', 'c']);
},
});
},
})(),
);
init.add(
createBackendModule({
pluginId: 'test',
moduleId: 'modC',
register(reg) {
reg.registerInit({
deps: { extension: extensionPoint },
async init({ extension }) {
expect(extension.values).toEqual(['b', 'a']);
extension.values.push('c');
},
});
},
})(),
);
await init.start();
});
it('should forward errors when plugins fail to start', async () => {
const init = new BackendInitializer(new ServiceRegistry([]));
init.add(
@@ -175,4 +250,48 @@ describe('BackendInitializer', () => {
"Module 'mod' for plugin 'test' is already registered",
);
});
it('should reject modules with circular dependencies', async () => {
const extA = createExtensionPoint<string>({ id: 'a' });
const extB = createExtensionPoint<string>({ id: 'b' });
const init = new BackendInitializer(
new ServiceRegistry([
rootLifecycleServiceFactory(),
createServiceFactory({
service: coreServices.rootLogger,
deps: {},
factory: () => new MockLogger(),
})(),
]),
);
init.add(
createBackendModule({
pluginId: 'test',
moduleId: 'modA',
register(reg) {
reg.registerExtensionPoint(extA, 'a');
reg.registerInit({
deps: { ext: extB },
async init() {},
});
},
})(),
);
init.add(
createBackendModule({
pluginId: 'test',
moduleId: 'modB',
register(reg) {
reg.registerExtensionPoint(extB, 'b');
reg.registerInit({
deps: { ext: extA },
async init() {},
});
},
})(),
);
await expect(init.start()).rejects.toThrow(
"Circular dependency detected for modules of plugin 'test', 'modA' -> 'modB' -> 'modA'",
);
});
});
@@ -26,8 +26,9 @@ import { EnumerableServiceHolder, ServiceOrExtensionPoint } from './types';
// Direct internal import to avoid duplication
// eslint-disable-next-line @backstage/no-forbidden-package-imports
import { InternalBackendFeature } from '@backstage/backend-plugin-api/src/wiring/types';
import { ForwardedError } from '@backstage/errors';
import { ForwardedError, ConflictError } from '@backstage/errors';
import { featureDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha';
import { DependencyGraph } from '../lib/DependencyGraph';
export interface BackendRegisterInit {
consumes: Set<ServiceOrExtensionPoint>;
@@ -161,7 +162,7 @@ export class BackendInitializer {
for (const r of feature.getRegistrations()) {
const provides = new Set<ExtensionPoint<unknown>>();
if (r.type === 'plugin') {
if (r.type === 'plugin' || r.type === 'module') {
for (const [extRef, extImpl] of r.extensionPoints) {
if (this.#extensionPoints.has(extRef)) {
throw new Error(
@@ -210,21 +211,41 @@ export class BackendInitializer {
await Promise.all(
allPluginIds.map(async pluginId => {
// Modules are initialized before plugins, so that they can provide extension to the plugin
const modules = moduleInits.get(pluginId) ?? [];
await Promise.all(
Array.from(modules).map(async ([moduleId, moduleInit]) => {
const moduleDeps = await this.#getInitDeps(
moduleInit.init.deps,
pluginId,
const modules = moduleInits.get(pluginId);
if (modules) {
const tree = DependencyGraph.fromIterable(
Array.from(modules).map(([moduleId, moduleInit]) => ({
value: { moduleId, moduleInit },
// Relationships are reversed at this point since we're only interested in the extension points.
// If a modules provides extension point A we want it to be initialized AFTER all modules
// that depend on extension point A, so that they can provide their extensions.
consumes: Array.from(moduleInit.provides).map(p => p.id),
provides: Array.from(moduleInit.consumes).map(c => c.id),
})),
);
const circular = tree.detectCircularDependency();
if (circular) {
throw new ConflictError(
`Circular dependency detected for modules of plugin '${pluginId}', ${circular
.map(({ moduleId }) => `'${moduleId}'`)
.join(' -> ')}`,
);
await moduleInit.init.func(moduleDeps).catch(error => {
throw new ForwardedError(
`Module '${moduleId}' for plugin '${pluginId}' startup failed`,
error,
}
await tree.parallelTopologicalTraversal(
async ({ moduleId, moduleInit }) => {
const moduleDeps = await this.#getInitDeps(
moduleInit.init.deps,
pluginId,
);
});
}),
);
await moduleInit.init.func(moduleDeps).catch(error => {
throw new ForwardedError(
`Module '${moduleId}' for plugin '${pluginId}' startup failed`,
error,
);
});
},
);
}
// Once all modules have been initialized, we can initialize the plugin itself
const pluginInit = pluginInits.get(pluginId);
@@ -31,6 +31,11 @@ export interface BackendModuleConfig {
// @public
export interface BackendModuleRegistrationPoints {
// (undocumented)
registerExtensionPoint<TExtensionPoint>(
ref: ExtensionPoint<TExtensionPoint>,
impl: TExtensionPoint,
): void;
// (undocumented)
registerInit<
Deps extends {
@@ -149,6 +149,7 @@ describe('createBackendModule', () => {
type: 'module',
pluginId: 'x',
moduleId: 'y',
extensionPoints: [],
init: {
deps: expect.any(Object),
func: expect.any(Function),
@@ -159,13 +159,14 @@ export function createBackendPlugin<TOptions extends [options?: object] = []>(
*/
export interface BackendModuleConfig {
/**
* The ID of this plugin.
* Should exactly match the `id` of the plugin that the module extends.
*
* @see {@link https://backstage.io/docs/backend-system/architecture/naming-patterns | Recommended naming patterns}
*/
pluginId: string;
/**
* Should exactly match the `id` of the plugin that the module extends.
* The ID of this module, used to identify the module and ensure that it is not installed twice.
*/
moduleId: string;
register(reg: BackendModuleRegistrationPoints): void;
@@ -194,10 +195,20 @@ export function createBackendModule<TOptions extends [options?: object] = []>(
if (registrations) {
return registrations;
}
const extensionPoints: InternalBackendPluginRegistration['extensionPoints'] =
[];
let init: InternalBackendModuleRegistration['init'] | undefined =
undefined;
c.register({
registerExtensionPoint(ext, impl) {
if (init) {
throw new Error(
'registerExtensionPoint called after registerInit',
);
}
extensionPoints.push([ext, impl]);
},
registerInit(regInit) {
if (init) {
throw new Error('registerInit must only be called once');
@@ -220,6 +231,7 @@ export function createBackendModule<TOptions extends [options?: object] = []>(
type: 'module',
pluginId: c.pluginId,
moduleId: c.moduleId,
extensionPoints,
init,
},
];
@@ -59,6 +59,10 @@ export interface BackendPluginRegistrationPoints {
* @public
*/
export interface BackendModuleRegistrationPoints {
registerExtensionPoint<TExtensionPoint>(
ref: ExtensionPoint<TExtensionPoint>,
impl: TExtensionPoint,
): void;
registerInit<Deps extends { [name in string]: unknown }>(options: {
deps: {
[name in keyof Deps]: ServiceRef<Deps[name]> | ExtensionPoint<Deps[name]>;
@@ -105,6 +109,7 @@ export interface InternalBackendModuleRegistration {
pluginId: string;
moduleId: string;
type: 'module';
extensionPoints: Array<readonly [ExtensionPoint<unknown>, unknown]>;
init: {
deps: Record<string, ServiceRef<unknown> | ExtensionPoint<unknown>>;
func(deps: Record<string, unknown>): Promise<void>;