From f3438696520ae81565b33113211e6b31ab6dfe3b Mon Sep 17 00:00:00 2001 From: David Festal Date: Tue, 13 Jun 2023 10:32:28 +0200 Subject: [PATCH 1/6] New `backend-plugin-manager` service that: - scans and imports plugins dynamically from a root folder, - provides plugin initialization, for both the legacy and the new backend system. Signed-off-by: David Festal Co-Authored-By: Gorkem Ercan --- packages/backend-plugin-manager/.eslintrc.js | 1 + packages/backend-plugin-manager/README.md | 60 ++ packages/backend-plugin-manager/api-report.md | 207 ++++++ packages/backend-plugin-manager/config.d.ts | 26 + packages/backend-plugin-manager/package.json | 63 ++ .../src/__testUtils__/testUtils.ts | 77 ++ packages/backend-plugin-manager/src/index.ts | 20 + .../src/loader/CommonJSModuleLoader.ts | 50 ++ .../src/loader/types.ts | 28 + .../src/manager/plugin-manager.test.ts | 535 ++++++++++++++ .../src/manager/plugin-manager.ts | 265 +++++++ .../src/manager/types.ts | 160 ++++ .../scanner/plugin-scanner-watcher.test.ts | 359 +++++++++ .../src/scanner/plugin-scanner.test.ts | 683 ++++++++++++++++++ .../src/scanner/plugin-scanner.ts | 302 ++++++++ .../src/scanner/types.ts | 37 + yarn.lock | 34 + 17 files changed, 2907 insertions(+) create mode 100644 packages/backend-plugin-manager/.eslintrc.js create mode 100644 packages/backend-plugin-manager/README.md create mode 100644 packages/backend-plugin-manager/api-report.md create mode 100644 packages/backend-plugin-manager/config.d.ts create mode 100644 packages/backend-plugin-manager/package.json create mode 100644 packages/backend-plugin-manager/src/__testUtils__/testUtils.ts create mode 100644 packages/backend-plugin-manager/src/index.ts create mode 100644 packages/backend-plugin-manager/src/loader/CommonJSModuleLoader.ts create mode 100644 packages/backend-plugin-manager/src/loader/types.ts create mode 100644 packages/backend-plugin-manager/src/manager/plugin-manager.test.ts create mode 100644 packages/backend-plugin-manager/src/manager/plugin-manager.ts create mode 100644 packages/backend-plugin-manager/src/manager/types.ts create mode 100644 packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts create mode 100644 packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts create mode 100644 packages/backend-plugin-manager/src/scanner/plugin-scanner.ts create mode 100644 packages/backend-plugin-manager/src/scanner/types.ts diff --git a/packages/backend-plugin-manager/.eslintrc.js b/packages/backend-plugin-manager/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/packages/backend-plugin-manager/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/backend-plugin-manager/README.md b/packages/backend-plugin-manager/README.md new file mode 100644 index 0000000000..42b68739a9 --- /dev/null +++ b/packages/backend-plugin-manager/README.md @@ -0,0 +1,60 @@ +# @backstage/backend-plugin-manager + +This package adds experimental support for **dynamic backend plugins**, according to the content of the proposal in [RFC 18390](https://github.com/backstage/backstage/issues/18390) + +## Status + +**This package is EXPERIMENTAL, and is subject to change according to the discussions and conclusions that happen around the RFC mentioned above.** + +## Testing the backend dynamic plugins feature + +In order to test the dynamic backend plugins feature provided by this package, example applications, as well as example dynamic plugins have been provided in provided in the [showcase repository](https://github.com/janus-idp/dynamic-backend-plugins-showcase), and instructions are provided in the [related Readme](https://github.com/janus-idp/dynamic-backend-plugins-showcase#readme). + +## How it works + +The backend plugin manager is a service that scans a configured root directory (`dynamicPlugins.rootDirectory` in the app config) for dynamic plugin packages, and loads them dynamically. + +In the `backend-next` application, it can be enabled by adding the `backend-plugin-manager` as a dependency in the `package.json` and the following lines in the `src/index.ts` file: + +```ts +- +-const backend = createBackend(); ++import { ++ dynamicPluginsServiceFactory, ++ dynamicPluginsFeatureDiscoveryServiceFactory, ++} from '@backstage/backend-plugin-manager'; ++ ++const backend = createBackend({ ++ services: [ ++ dynamicPluginsServiceFactory(), ++ dynamicPluginsFeatureDiscoveryServiceFactory(), // overridden version of the FeatureDiscoveryService which provides features loaded by dynamic plugins ++ ], ++}); +``` + +### About the expected dynamic plugin structure + +Due to some limitations of the current backstage codebase, the plugins need to be completed and repackaged to by used as dynamic plugins: + +1. they must provide a named entry point (`dynamicPluginInstaller`) of a specific type (`BackendDynamicPluginInstaller`), as can be found in the `src/dynamic` sub-folder of each dynamic plugin example provided in the [showcase repository](https://github.com/janus-idp/dynamic-backend-plugins-showcase). +2. they would have a modified `package.json` file in which dependencies are updated to share `@backstage` dependencies with the main application. +3. they may embed some dependency whose code is then merged with the plugin code. + +Points 2 and 3 can be done by the `export-dynamic-plugin` CLI command used to perform the repackaging + +### About the `export-dynamic-plugin` command + +The `export-dynamic-plugin` CLI command, used the dynamic plugin examples provided in the [showcase repository](https://github.com/janus-idp/dynamic-backend-plugins-showcase), is part of a `@backstage/cli` fork (`@dfatwork-pkgs/backstage-cli@0.22.9-next.6`), and can be used to help packaging the dynamic plugins according to the constraints mentioned above, in order to allow straightforward testing of the dynamic plugins feature. + +However the `backend-plugin-manager` experimental package does **not** depend on the use of this additional CLI command, and in future steps of this backend dynamic plugin work, the use of such a dedicated command might not even be necessary. + +### About the support of the legacy backend system + +The backend dynamic plugins feature clearly targets the new backend system. +However some level of compatibility is provided with current backstage codebase, which still uses the legacy backend system, in order to allow testing and exploring dynamic backend plugin support on the widest range of contexts and installations. +However, this is temporary and will be removed once the next backend is ready to be used and has completely replaced the old one. +This is why the API related to the old backend is already marked as deprecated. + +### Future work + +The current implementation of the backend plugin manager is a first step towards the final implementation of the dynamic backend plugins feature, which will be completed / simplified in future steps, as the status of the backstage codebase allows it. diff --git a/packages/backend-plugin-manager/api-report.md b/packages/backend-plugin-manager/api-report.md new file mode 100644 index 0000000000..e10f71ba15 --- /dev/null +++ b/packages/backend-plugin-manager/api-report.md @@ -0,0 +1,207 @@ +## API Report File for "@backstage/backend-plugin-manager" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { EventBroker } from '@backstage/plugin-events-node'; +import { EventsBackend } from '@backstage/plugin-events-backend'; +import { FeatureDiscoveryService } from '@backstage/backend-plugin-api/alpha'; +import { HttpPostIngressOptions } from '@backstage/plugin-events-node'; +import { IdentityApi } from '@backstage/plugin-auth-node'; +import { IndexBuilder } from '@backstage/plugin-search-backend-node'; +import { Logger } from 'winston'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { PackagePlatform } from '@backstage/cli-node'; +import { PackageRole } from '@backstage/cli-node'; +import { PermissionEvaluator } from '@backstage/plugin-permission-common'; +import { PermissionPolicy } from '@backstage/plugin-permission-node'; +import { PluginCacheManager } from '@backstage/backend-common'; +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Router } from 'express'; +import { ServiceFactory } from '@backstage/backend-plugin-api'; +import { ServiceRef } from '@backstage/backend-plugin-api'; +import { TaskRunner } from '@backstage/backend-tasks'; +import { TemplateAction } from '@backstage/plugin-scaffolder-node'; +import { TokenManager } from '@backstage/backend-common'; +import { UrlReader } from '@backstage/backend-common'; + +// @public (undocumented) +export interface BackendDynamicPlugin extends BaseDynamicPlugin { + // (undocumented) + installer: BackendDynamicPluginInstaller; + // (undocumented) + platform: 'node'; +} + +// @public (undocumented) +export type BackendDynamicPluginInstaller = + | LegacyBackendPluginInstaller + | NewBackendPluginInstaller; + +// @public (undocumented) +export interface BackendPluginProvider { + // (undocumented) + backendPlugins(): BackendDynamicPlugin[]; +} + +// @public (undocumented) +export interface BaseDynamicPlugin { + // (undocumented) + name: string; + // (undocumented) + platform: PackagePlatform; + // (undocumented) + role: PackageRole; + // (undocumented) + version: string; +} + +// @public (undocumented) +export type DynamicPlugin = FrontendDynamicPlugin | BackendDynamicPlugin; + +// @public (undocumented) +export interface DynamicPluginsFactoryOptions { + // (undocumented) + moduleLoader?(logger: LoggerService): ModuleLoader; +} + +// @public (undocumented) +export const dynamicPluginsFeatureDiscoveryServiceFactory: () => ServiceFactory< + FeatureDiscoveryService, + 'root' +>; + +// @public (undocumented) +export const dynamicPluginsServiceFactory: ( + options?: DynamicPluginsFactoryOptions | undefined, +) => ServiceFactory; + +// @public (undocumented) +export const dynamicPluginsServiceRef: ServiceRef< + BackendPluginProvider, + 'root' +>; + +// @public (undocumented) +export interface FrontendDynamicPlugin extends BaseDynamicPlugin { + // (undocumented) + platform: 'web'; +} + +// @public (undocumented) +export function isBackendDynamicPluginInstaller( + obj: any, +): obj is BackendDynamicPluginInstaller; + +// @public @deprecated (undocumented) +export interface LegacyBackendPluginInstaller { + // (undocumented) + catalog?(builder: CatalogBuilder, env: PluginEnvironment): void; + // (undocumented) + events?( + eventsBackend: EventsBackend, + env: PluginEnvironment, + ): HttpPostIngressOptions[]; + // (undocumented) + kind: 'legacy'; + // (undocumented) + permissions?: { + policy?: PermissionPolicy; + }; + // (undocumented) + router?: { + pluginID: string; + createPlugin(env: PluginEnvironment): Promise; + }; + // (undocumented) + scaffolder?(env: PluginEnvironment): TemplateAction[]; + // (undocumented) + search?( + indexBuilder: IndexBuilder, + schedule: TaskRunner, + env: PluginEnvironment, + ): void; +} + +// @public (undocumented) +export interface ModuleLoader { + // (undocumented) + bootstrap(backstageRoot: string, dynamicPluginPaths: string[]): Promise; + // (undocumented) + load(id: string): Promise; + // (undocumented) + logger: LoggerService; +} + +// @public (undocumented) +export interface NewBackendPluginInstaller { + // (undocumented) + install(): BackendFeature | BackendFeature[]; + // (undocumented) + kind: 'new'; +} + +// @public (undocumented) +export type PluginEnvironment = { + logger: Logger; + cache: PluginCacheManager; + database: PluginDatabaseManager; + config: Config; + reader: UrlReader; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + permissions: PermissionEvaluator; + scheduler: PluginTaskScheduler; + identity: IdentityApi; + eventBroker: EventBroker; + pluginProvider: BackendPluginProvider; +}; + +// @public (undocumented) +export class PluginManager implements BackendPluginProvider { + // (undocumented) + addBackendPlugin(plugin: BackendDynamicPlugin): void; + // (undocumented) + get availablePackages(): ScannedPluginPackage[]; + // (undocumented) + backendPlugins(): BackendDynamicPlugin[]; + // (undocumented) + static fromConfig( + config: Config, + logger: LoggerService, + preferAlpha?: boolean, + mooduleLoader?: ModuleLoader, + ): Promise; + // (undocumented) + readonly plugins: DynamicPlugin[]; +} + +// @public (undocumented) +export interface ScannedPluginManifest { + // (undocumented) + backstage: { + role: PackageRole; + }; + // (undocumented) + main: string; + // (undocumented) + name: string; + // (undocumented) + version: string; +} + +// @public (undocumented) +export interface ScannedPluginPackage { + // (undocumented) + location: URL; + // (undocumented) + manifest: ScannedPluginManifest; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/packages/backend-plugin-manager/config.d.ts b/packages/backend-plugin-manager/config.d.ts new file mode 100644 index 0000000000..edbc99ce89 --- /dev/null +++ b/packages/backend-plugin-manager/config.d.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** @visibility frontend */ + dynamicPlugins?: { + /** + * The local path, relative to the backstage root, that contains dynamic plugins. + * @visibility frontend + */ + rootDirectory: string; + }; +} diff --git a/packages/backend-plugin-manager/package.json b/packages/backend-plugin-manager/package.json new file mode 100644 index 0000000000..12bfd179fb --- /dev/null +++ b/packages/backend-plugin-manager/package.json @@ -0,0 +1,63 @@ +{ + "name": "@backstage/backend-plugin-manager", + "packageManager": "yarn@3.2.3", + "description": "Backstage plugin management backend", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "node-library" + }, + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean", + "start": "backstage-cli package start" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/cli-common": "workspace:^", + "@backstage/cli-node": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/plugin-auth-node": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-events-backend": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", + "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-permission-node": "workspace:^", + "@backstage/plugin-scaffolder-node": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "@backstage/plugin-search-common": "workspace:^", + "@backstage/types": "workspace:^", + "@types/express": "^4.17.6", + "chokidar": "^3.5.3", + "express": "^4.17.1", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-app-api": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/config-loader": "workspace:^", + "mock-fs": "^5.2.0", + "wait-for-expect": "^3.0.2" + }, + "files": [ + "dist" + ] +} diff --git a/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts b/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts new file mode 100644 index 0000000000..57bd9e4973 --- /dev/null +++ b/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { JsonObject } from '@backstage/types'; + +export type logContent = { message: string; meta?: Error | JsonObject }; + +export type Logs = { + errors?: logContent[]; + warns?: logContent[]; + infos?: logContent[]; + debugs?: logContent[]; +}; + +export class MockedLogger implements LoggerService { + logs: Logs = {}; + + error(message: string, meta?: Error | JsonObject): void { + if (!this.logs.errors) { + this.logs.errors = []; + } + this.logs.errors!.push(toLogContent(message, meta)); + } + warn(message: string, meta?: Error | JsonObject): void { + if (!this.logs.warns) { + this.logs.warns = []; + } + this.logs.warns!.push(toLogContent(message, meta)); + } + info(message: string, meta?: Error | JsonObject): void { + if (!this.logs.infos) { + this.logs.infos = []; + } + this.logs.infos!.push(toLogContent(message, meta)); + } + debug(message: string, meta?: Error | JsonObject): void { + if (!this.logs.debugs) { + this.logs.debugs = []; + } + this.logs.debugs!.push(toLogContent(message, meta)); + } + + child(_: JsonObject): LoggerService { + return this; + } +} + +function toLogContent(message: string, meta?: Error | JsonObject): logContent { + if (!meta) { + return { message }; + } + const jsonObject = Object.fromEntries(Object.entries(meta)); + if ('name' in meta) { + jsonObject.name = meta.name; + } + if ('message' in meta) { + jsonObject.message = meta.message; + } + return { + message, + meta: jsonObject, + }; +} diff --git a/packages/backend-plugin-manager/src/index.ts b/packages/backend-plugin-manager/src/index.ts new file mode 100644 index 0000000000..997913454e --- /dev/null +++ b/packages/backend-plugin-manager/src/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './scanner/types'; +export * from './loader/types'; +export * from './manager/types'; +export * from './manager/plugin-manager'; diff --git a/packages/backend-plugin-manager/src/loader/CommonJSModuleLoader.ts b/packages/backend-plugin-manager/src/loader/CommonJSModuleLoader.ts new file mode 100644 index 0000000000..dd64dc78f3 --- /dev/null +++ b/packages/backend-plugin-manager/src/loader/CommonJSModuleLoader.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ModuleLoader } from './types'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import path from 'path'; + +export class CommonJSModuleLoader implements ModuleLoader { + constructor(public readonly logger: LoggerService) {} + + async bootstrap( + backstageRoot: string, + dynamicPluginsPaths: string[], + ): Promise { + const allowedNodeModulesPaths = [ + `${backstageRoot}/node_modules`, + ...dynamicPluginsPaths.map(p => path.resolve(p, 'node_modules')), + ]; + const Module = require('module'); + const oldNodeModulePaths = Module._nodeModulePaths; + Module._nodeModulePaths = (from: string): string[] => { + const result: string[] = oldNodeModulePaths(from); + if (!dynamicPluginsPaths.some(p => from.startsWith(p))) { + return result; + } + + const filtered = result.filter(p => allowedNodeModulesPaths.includes(p)); + this.logger.debug( + `Overriding node_modules search path for dynamic plugin ${from} to: ${filtered}`, + ); + return filtered; + }; + } + + async load(packagePath: string): Promise { + return await import(/* webpackIgnore: true */ packagePath); + } +} diff --git a/packages/backend-plugin-manager/src/loader/types.ts b/packages/backend-plugin-manager/src/loader/types.ts new file mode 100644 index 0000000000..ebd3a1e7ce --- /dev/null +++ b/packages/backend-plugin-manager/src/loader/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; + +/** + * @public + */ +export interface ModuleLoader { + logger: LoggerService; + + bootstrap(backstageRoot: string, dynamicPluginPaths: string[]): Promise; + + load(id: string): Promise; +} diff --git a/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts new file mode 100644 index 0000000000..2d0966abbe --- /dev/null +++ b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts @@ -0,0 +1,535 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginManager, dynamicPluginsServiceFactory } from './plugin-manager'; +import { + BackendFeature, + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import mockFs, { directory, symlink } from 'mock-fs'; +import * as path from 'path'; +import * as url from 'url'; + +import { + BackendDynamicPlugin, + BaseDynamicPlugin, + DynamicPlugin, + LegacyBackendPluginInstaller, + NewBackendPluginInstaller, + PluginEnvironment, +} from './types'; +import { ScannedPluginManifest, ScannedPluginPackage } from '../scanner/types'; +import { randomUUID } from 'crypto'; +import { TemplateAction } from '@backstage/plugin-scaffolder-node'; +import { + createSpecializedBackend, + rootLifecycleServiceFactory, +} from '@backstage/backend-app-api'; +import { ConfigSources } from '@backstage/config-loader'; +import { Logs, MockedLogger, logContent } from '../__testUtils__/testUtils'; +import { PluginScanner } from '../scanner/plugin-scanner'; +import { findPaths } from '@backstage/cli-common'; + +describe('backend-plugin-manager', () => { + describe('loadPlugins', () => { + afterEach(() => { + mockFs.restore(); + jest.resetModules(); + }); + + type TestCase = { + name: string; + packageManifest: ScannedPluginManifest; + indexFile?: { + retativePath: string[]; + content?: string; + }; + expectedLogs?(location: URL): { + errors?: logContent[]; + warns?: logContent[]; + infos?: logContent[]; + debugs?: logContent[]; + }; + checkLoadedPlugins: (loadedPlugins: BaseDynamicPlugin[]) => void; + }; + + it.each([ + { + name: 'should successfully load a new backend plugin', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: + 'exports.dynamicPluginInstaller={ kind: "new", install: () => [] }', + }, + expectedLogs(location) { + return { + infos: [ + { + message: `loaded dynamic backend plugin 'backend-dynamic-plugin-test' from '${location}'`, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([ + { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + role: 'backend-plugin', + platform: 'node', + installer: { + kind: 'new', + }, + }, + ]); + const installer: NewBackendPluginInstaller = ( + plugins[0] as BackendDynamicPlugin + ).installer as NewBackendPluginInstaller; + expect(installer.install()).toEqual< + BackendFeature | BackendFeature[] + >([]); + }, + }, + { + name: 'should successfully load a new backend plugin module', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin-module', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: + 'exports.dynamicPluginInstaller={ kind: "new", install: () => [] }', + }, + expectedLogs(location) { + return { + infos: [ + { + message: `loaded dynamic backend plugin 'backend-dynamic-plugin-test' from '${location}'`, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([ + { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + role: 'backend-plugin-module', + platform: 'node', + installer: { + kind: 'new', + }, + }, + ]); + const installer: NewBackendPluginInstaller = ( + plugins[0] as BackendDynamicPlugin + ).installer as NewBackendPluginInstaller; + expect(installer.install()).toEqual< + BackendFeature | BackendFeature[] + >([]); + }, + }, + { + name: 'should fail when no index file', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + expectedLogs(location) { + const loadOrigin = 'manager/plugin-manager.test.ts'; + /* + if (process.env.CI === 'true') { + // CI runs the tests in a different way which produces a different relative path + loadOrigin = + '../../../packages/backend-plugin-manager/src/manager/plugin-manager.test.ts'; + } +*/ + return { + errors: [ + { + message: `an error occured while loading dynamic backend plugin 'backend-dynamic-plugin-test' from '${location}'`, + meta: { + name: 'Error', + message: `Cannot find module '${url.fileURLToPath( + location, + )}/dist/index.cjs.js' from '${loadOrigin}'`, + _originalMessage: `Cannot find module '${url.fileURLToPath( + location, + )}/dist/index.cjs.js' from '${loadOrigin}'`, + code: 'MODULE_NOT_FOUND', + hint: '', + moduleName: `${url.fileURLToPath( + location, + )}/dist/index.cjs.js`, + siblingWithSimilarExtensionFound: false, + requireStack: undefined, + }, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([]); + }, + }, + { + name: 'should fail when the expected entry point is not in the index file', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: '', + }, + expectedLogs(location) { + return { + errors: [ + { + message: `dynamic backend plugin 'backend-dynamic-plugin-test' could not be loaded from '${location}': the module should export a 'const dynamicPluginInstaller: BackendDynamicPluginInstaller' field.`, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([]); + }, + }, + { + name: 'should fail when the expected entry point is not of the expected type', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: + 'exports.dynamicPluginInstaller={ something: "else", unexpectedMethod() {} }', + }, + expectedLogs(location) { + return { + errors: [ + { + message: `dynamic backend plugin 'backend-dynamic-plugin-test' could not be loaded from '${location}': the module should export a 'const dynamicPluginInstaller: BackendDynamicPluginInstaller' field.`, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([]); + }, + }, + { + name: 'should fail when the index file has a syntax error', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: 'strange text with syntax error', + }, + expectedLogs(location) { + return { + errors: [ + { + message: `an error occured while loading dynamic backend plugin 'backend-dynamic-plugin-test' from '${location}'`, + meta: { + message: 'Unexpected identifier', + name: 'SyntaxError', + }, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([]); + }, + }, + { + name: 'should successfully load a legacy backend plugin', + packageManifest: { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + }, + indexFile: { + retativePath: ['dist', 'index.cjs.js'], + content: + 'exports.dynamicPluginInstaller={ kind: "legacy", scaffolder: (env)=>[] }', + }, + expectedLogs(location) { + return { + infos: [ + { + message: `loaded dynamic backend plugin 'backend-dynamic-plugin-test' from '${location}'`, + }, + ], + }; + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([ + { + name: 'backend-dynamic-plugin-test', + version: '0.0.0', + role: 'backend-plugin', + platform: 'node', + installer: { + kind: 'legacy', + }, + }, + ]); + const installer = (plugins[0] as BackendDynamicPlugin) + .installer as LegacyBackendPluginInstaller; + expect(installer.scaffolder!({} as PluginEnvironment)).toEqual< + TemplateAction[] + >([]); + }, + }, + { + name: 'should successfully load a frontend plugin', + packageManifest: { + name: 'frontend-dynamic-plugin-test', + version: '0.0.0', + backstage: { + role: 'frontend-plugin', + }, + main: 'dist/index.esm.js', + }, + checkLoadedPlugins(plugins) { + expect(plugins).toMatchObject([ + { + name: 'frontend-dynamic-plugin-test', + version: '0.0.0', + role: 'frontend-plugin', + platform: 'web', + }, + ]); + }, + }, + ])('$name', async (tc: TestCase): Promise => { + const plugin: ScannedPluginPackage = { + location: new URL(`file:///node_modules/jest-tests/${randomUUID()}`), + manifest: tc.packageManifest, + }; + + const mockedFiles = { + [path.join(url.fileURLToPath(plugin.location), 'package.json')]: + mockFs.file({ + content: JSON.stringify(plugin), + }), + }; + if (tc.indexFile) { + mockedFiles[ + path.join( + url.fileURLToPath(plugin.location), + ...tc.indexFile.retativePath, + ) + ] = mockFs.file({ + content: tc.indexFile.content, + }); + } + mockFs(mockedFiles); + + const logger = new MockedLogger(); + const pluginManager = new (PluginManager as any)(logger, [plugin], { + logger, + async bootstrap(_: string, __: string[]): Promise {}, + load: async (packagePath: string) => + await import(/* webpackIgnore: true */ packagePath), + }); + + const loadedPlugins: DynamicPlugin[] = await pluginManager.loadPlugins(); + + const expectedLogs = tc.expectedLogs + ? tc.expectedLogs(plugin.location) + : {}; + expect(logger.logs).toEqual(expectedLogs); + + tc.checkLoadedPlugins(loadedPlugins); + }); + }); + + describe('backendPlugins', () => { + it('should return only backend plugins and modules', async () => { + const logger = new MockedLogger(); + const pluginManager = new (PluginManager as any)(logger, '', []); + const plugins: BaseDynamicPlugin[] = [ + { + name: 'a-frontend-plugin', + platform: 'web', + role: 'frontend-plugin', + version: '0.0.0', + }, + { + name: 'a-backend-plugin', + platform: 'node', + role: 'backend-plugin', + version: '0.0.0', + }, + { + name: 'a-backend-module', + platform: 'node', + role: 'backend-plugin-module', + version: '0.0.0', + }, + ]; + pluginManager.plugins = plugins; + expect(pluginManager.backendPlugins()).toEqual([ + { + name: 'a-backend-plugin', + platform: 'node', + role: 'backend-plugin', + version: '0.0.0', + }, + { + name: 'a-backend-module', + platform: 'node', + role: 'backend-plugin-module', + version: '0.0.0', + }, + ]); + }); + }); + + describe('dynamicPluginsServiceFactory', () => { + afterEach(() => { + mockFs.restore(); + jest.resetModules(); + }); + + it('should call PluginManager.fromConfig', async () => { + const logger = new MockedLogger(); + const rootLogger = new MockedLogger(); + + mockFs({ + [findPaths(__dirname).resolveTargetRoot('package.json')]: mockFs.load( + findPaths(__dirname).resolveTargetRoot('package.json'), + ), + '/somewhere/dynamic-plugins-root/a-dynamic-plugin': symlink({ + path: '/somewhere-else/a-dynamic-plugin', + }), + '/somewhere-else/a-dynamic-plugin': directory({}), + }); + + const fromConfigSpier = jest.spyOn(PluginManager, 'fromConfig'); + const applyConfigSpier = jest + .spyOn(PluginScanner.prototype as any, 'applyConfig') + .mockImplementation(() => {}); + const scanRootSpier = jest + .spyOn(PluginScanner.prototype, 'scanRoot') + .mockImplementation(async () => [ + { + location: new URL( + 'file:///somewhere/dynamic-plugins-root/a-dynamic-plugin', + ), + manifest: { + name: 'test', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { + role: 'backend-plugin', + }, + }, + }, + ]); + const mockedModuleLoader = { + logger, + bootstrap: jest.fn(), + load: jest.fn(), + }; + + const backend = createSpecializedBackend({ + services: [ + rootLifecycleServiceFactory(), + createServiceFactory({ + service: coreServices.rootConfig, + deps: {}, + async factory({}) { + return await ConfigSources.toConfig({ + async *readConfigData() { + yield { configs: [] }; + }, + }); + }, + }), + createServiceFactory({ + service: coreServices.logger, + deps: {}, + async factory({}) { + return logger; + }, + }), + createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + async factory({}) { + return rootLogger; + }, + }), + dynamicPluginsServiceFactory({ + moduleLoader: _ => mockedModuleLoader, + }), + ], + }); + + await backend.start(); + expect(fromConfigSpier).toHaveBeenCalled(); + expect(applyConfigSpier).toHaveBeenCalled(); + expect(scanRootSpier).toHaveBeenCalled(); + expect(mockedModuleLoader.bootstrap).toHaveBeenCalledWith( + findPaths(__dirname).targetRoot, + ['/somewhere-else/a-dynamic-plugin'], + ); + expect(mockedModuleLoader.load).toHaveBeenCalledWith( + '/somewhere/dynamic-plugins-root/a-dynamic-plugin/dist/index.cjs.js', + ); + }); + }); +}); diff --git a/packages/backend-plugin-manager/src/manager/plugin-manager.ts b/packages/backend-plugin-manager/src/manager/plugin-manager.ts new file mode 100644 index 0000000000..b4cfaeeb64 --- /dev/null +++ b/packages/backend-plugin-manager/src/manager/plugin-manager.ts @@ -0,0 +1,265 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config } from '@backstage/config'; +import { + BackendPluginProvider, + BackendDynamicPlugin, + isBackendDynamicPluginInstaller, + DynamicPlugin, +} from './types'; +import { PluginScanner } from '../scanner/plugin-scanner'; +import * as url from 'url'; +import { ScannedPluginPackage } from '../scanner/types'; +import { + BackendFeature, + LoggerService, + coreServices, + createServiceFactory, + createServiceRef, +} from '@backstage/backend-plugin-api'; +import { PackageRoles } from '@backstage/cli-node'; +import { findPaths } from '@backstage/cli-common'; +import path from 'path'; +import * as fs from 'fs'; +import { ModuleLoader } from '../loader/types'; +import { CommonJSModuleLoader } from '../loader/CommonJSModuleLoader'; +import { + FeatureDiscoveryService, + featureDiscoveryServiceRef, +} from '@backstage/backend-plugin-api/alpha'; + +/** + * @public + */ +export class PluginManager implements BackendPluginProvider { + static async fromConfig( + config: Config, + logger: LoggerService, + preferAlpha: boolean = false, + mooduleLoader: ModuleLoader = new CommonJSModuleLoader(logger), + ): Promise { + /* eslint-disable-next-line no-restricted-syntax */ + const backstageRoot = findPaths(__dirname).targetRoot; + const scanner = new PluginScanner( + config, + logger, + backstageRoot, + preferAlpha, + ); + const scannedPlugins = await scanner.scanRoot(); + scanner.trackChanges(); + const manager = new PluginManager(logger, scannedPlugins, mooduleLoader); + + const dynamicPluginsPaths = scannedPlugins.map(p => + fs.realpathSync( + path.dirname( + path.dirname( + path.resolve(url.fileURLToPath(p.location), p.manifest.main), + ), + ), + ), + ); + + mooduleLoader.bootstrap(backstageRoot, dynamicPluginsPaths); + + scanner.subscribeToRootDirectoryChange(async () => { + manager._availablePackages = await scanner.scanRoot(); + // TODO: do not store _scannedPlugins again, but instead store a diff of the changes + }); + manager.plugins.push(...(await manager.loadPlugins())); + + return manager; + } + + readonly plugins: DynamicPlugin[]; + private _availablePackages: ScannedPluginPackage[]; + + private constructor( + private readonly logger: LoggerService, + private packages: ScannedPluginPackage[], + private readonly moduleLoader: ModuleLoader, + ) { + this.plugins = []; + this._availablePackages = packages; + } + + get availablePackages(): ScannedPluginPackage[] { + return this._availablePackages; + } + + addBackendPlugin(plugin: BackendDynamicPlugin): void { + this.plugins.push(plugin); + } + + private async loadPlugins(): Promise { + const loadedPlugins: DynamicPlugin[] = []; + + for (const scannedPlugin of this.packages) { + const platform = PackageRoles.getRoleInfo( + scannedPlugin.manifest.backstage.role, + ).platform; + + if ( + platform === 'node' && + scannedPlugin.manifest.backstage.role.includes('-plugin') + ) { + const plugin = await this.loadBackendPlugin(scannedPlugin); + if (plugin !== undefined) { + loadedPlugins.push(plugin); + } + } else { + loadedPlugins.push({ + name: scannedPlugin.manifest.name, + version: scannedPlugin.manifest.version, + role: scannedPlugin.manifest.backstage.role, + platform: 'web', + // TODO(davidfestal): add required front-end plugin information here. + }); + } + } + return loadedPlugins; + } + + private async loadBackendPlugin( + plugin: ScannedPluginPackage, + ): Promise { + const packagePath = url.fileURLToPath( + `${plugin.location}/${plugin.manifest.main}`, + ); + try { + const { dynamicPluginInstaller } = await this.moduleLoader.load( + packagePath, + ); + if (!isBackendDynamicPluginInstaller(dynamicPluginInstaller)) { + this.logger.error( + `dynamic backend plugin '${plugin.manifest.name}' could not be loaded from '${plugin.location}': the module should export a 'const dynamicPluginInstaller: BackendDynamicPluginInstaller' field.`, + ); + return undefined; + } + this.logger.info( + `loaded dynamic backend plugin '${plugin.manifest.name}' from '${plugin.location}'`, + ); + return { + name: plugin.manifest.name, + version: plugin.manifest.version, + platform: 'node', + role: plugin.manifest.backstage.role, + installer: dynamicPluginInstaller, + }; + } catch (error) { + this.logger.error( + `an error occured while loading dynamic backend plugin '${plugin.manifest.name}' from '${plugin.location}'`, + error, + ); + return undefined; + } + } + + backendPlugins(): BackendDynamicPlugin[] { + return this.plugins.filter( + (p): p is BackendDynamicPlugin => p.platform === 'node', + ); + } +} + +/** + * @public + */ +export const dynamicPluginsServiceRef = createServiceRef( + { + id: 'core.dynamicplugins', + scope: 'root', + }, +); + +/** + * @public + */ +export interface DynamicPluginsFactoryOptions { + moduleLoader?(logger: LoggerService): ModuleLoader; +} + +/** + * @public + */ +export const dynamicPluginsServiceFactory = createServiceFactory( + (options?: DynamicPluginsFactoryOptions) => ({ + service: dynamicPluginsServiceRef, + deps: { + config: coreServices.rootConfig, + logger: coreServices.rootLogger, + }, + async factory({ config, logger }) { + if (options?.moduleLoader) { + return await PluginManager.fromConfig( + config, + logger, + true, + options.moduleLoader(logger), + ); + } + return await PluginManager.fromConfig(config, logger, true); + }, + }), +); + +class DynamicPluginsEnabledFeatureDiscoveryService + implements FeatureDiscoveryService +{ + constructor( + private readonly dynamicPlugins: BackendPluginProvider, + private readonly featureDiscoveryService?: FeatureDiscoveryService, + ) {} + + async getBackendFeatures(): Promise<{ features: Array }> { + const staticFeatures = + (await this.featureDiscoveryService?.getBackendFeatures())?.features ?? + []; + + return { + features: [ + ...this.dynamicPlugins + .backendPlugins() + .flatMap((plugin): BackendFeature[] => { + if (plugin.installer.kind === 'new') { + const installed = plugin.installer.install(); + if (Array.isArray(installed)) { + return installed; + } + return [installed]; + } + return []; + }), + ...staticFeatures, + ], + }; + } +} + +/** + * @public + */ +export const dynamicPluginsFeatureDiscoveryServiceFactory = + createServiceFactory({ + service: featureDiscoveryServiceRef, + deps: { + config: coreServices.rootConfig, + dynamicPlugins: dynamicPluginsServiceRef, + }, + factory({ dynamicPlugins }) { + return new DynamicPluginsEnabledFeatureDiscoveryService(dynamicPlugins); + }, + }); diff --git a/packages/backend-plugin-manager/src/manager/types.ts b/packages/backend-plugin-manager/src/manager/types.ts new file mode 100644 index 0000000000..8c220b7699 --- /dev/null +++ b/packages/backend-plugin-manager/src/manager/types.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; +import { + PluginCacheManager, + PluginDatabaseManager, + PluginEndpointDiscovery, + TokenManager, + UrlReader, +} from '@backstage/backend-common'; +import { Router } from 'express'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import { IdentityApi } from '@backstage/plugin-auth-node'; +import { PermissionEvaluator } from '@backstage/plugin-permission-common'; +import { + EventBroker, + HttpPostIngressOptions, +} from '@backstage/plugin-events-node'; + +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { PackagePlatform, PackageRole } from '@backstage/cli-node'; +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +import { TemplateAction } from '@backstage/plugin-scaffolder-node'; +import { IndexBuilder } from '@backstage/plugin-search-backend-node'; +import { EventsBackend } from '@backstage/plugin-events-backend'; +import { PermissionPolicy } from '@backstage/plugin-permission-node'; + +/** + * @public + */ +export type PluginEnvironment = { + logger: Logger; + cache: PluginCacheManager; + database: PluginDatabaseManager; + config: Config; + reader: UrlReader; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + permissions: PermissionEvaluator; + scheduler: PluginTaskScheduler; + identity: IdentityApi; + eventBroker: EventBroker; + pluginProvider: BackendPluginProvider; +}; + +/** + * @public + */ +export interface BackendPluginProvider { + backendPlugins(): BackendDynamicPlugin[]; +} + +/** + * @public + */ +export interface BaseDynamicPlugin { + name: string; + version: string; + role: PackageRole; + platform: PackagePlatform; +} + +/** + * @public + */ +export type DynamicPlugin = FrontendDynamicPlugin | BackendDynamicPlugin; + +/** + * @public + */ +export interface FrontendDynamicPlugin extends BaseDynamicPlugin { + platform: 'web'; +} + +/** + * @public + */ +export interface BackendDynamicPlugin extends BaseDynamicPlugin { + platform: 'node'; + installer: BackendDynamicPluginInstaller; +} + +/** + * @public + */ +export type BackendDynamicPluginInstaller = + | LegacyBackendPluginInstaller + | NewBackendPluginInstaller; + +/** + * @public + */ +export interface NewBackendPluginInstaller { + kind: 'new'; + + install(): BackendFeature | BackendFeature[]; +} + +/** + * @public + * @deprecated + * + * Support for the legacy backend suystem will be removed in the future. + * + * When adding a legacy plugin installer entrypoint in your plugin, + * you should always take the opportunity to also implement support + * for the new backend system if not already done. + * + */ +export interface LegacyBackendPluginInstaller { + kind: 'legacy'; + + router?: { + pluginID: string; + createPlugin(env: PluginEnvironment): Promise; + }; + + catalog?(builder: CatalogBuilder, env: PluginEnvironment): void; + scaffolder?(env: PluginEnvironment): TemplateAction[]; + search?( + indexBuilder: IndexBuilder, + schedule: TaskRunner, + env: PluginEnvironment, + ): void; + events?( + eventsBackend: EventsBackend, + env: PluginEnvironment, + ): HttpPostIngressOptions[]; + permissions?: { + policy?: PermissionPolicy; + }; +} + +/** + * @public + */ +export function isBackendDynamicPluginInstaller( + obj: any, +): obj is BackendDynamicPluginInstaller { + return ( + obj !== undefined && + 'kind' in obj && + (obj.kind === 'new' || obj.kind === 'legacy') + ); +} diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts new file mode 100644 index 0000000000..2644024cd7 --- /dev/null +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts @@ -0,0 +1,359 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginScanner } from './plugin-scanner'; +import { Logs, MockedLogger } from '../__testUtils__/testUtils'; +import { Config, ConfigReader } from '@backstage/config'; +import path, { join } from 'path'; +import { ScannedPluginPackage } from './types'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import waitForExpect from 'wait-for-expect'; + +describe('plugin-scanner', () => { + describe('rootDirectoryWatcher', () => { + let backstageRootDirectory: string; + beforeEach(() => { + backstageRootDirectory = mkdtempSync( + path.join(tmpdir(), 'backstage-plugin-scanner-test'), + ); + }); + afterEach(() => { + if (backstageRootDirectory) { + rmSync(backstageRootDirectory, { recursive: true }); + } + }); + + it('watch for config and file system changes', async () => { + const logger = new MockedLogger(); + + mkdirSync(join(backstageRootDirectory, 'first-dir')); + mkdirSync(join(backstageRootDirectory, 'second-dir')); + + const config: Config = new ConfigReader({ + dynamicPlugins: { + rootDirectory: 'first-dir', + }, + }); + const getOptional = jest.spyOn(config, 'getOptional'); + + let onConfigChange: (() => Promise) | undefined; + const configUnsubscribe = jest.fn(); + config.subscribe = onChange => { + onConfigChange = onChange as any; + return { + unsubscribe: configUnsubscribe, + }; + }; + const pluginScanner = new PluginScanner( + config, + logger, + backstageRootDirectory, + false, + ); + await pluginScanner.trackChanges(); + + expect(onConfigChange).toBeDefined(); + + let scannedPlugins: ScannedPluginPackage[] = + await pluginScanner.scanRoot(); + expect(scannedPlugins).toEqual([]); + + const rootDirectorySubscriber = jest.fn(async () => { + scannedPlugins = await pluginScanner.scanRoot(); + }); + pluginScanner.subscribeToRootDirectoryChange(rootDirectorySubscriber); + + mkdirSync( + join(backstageRootDirectory, 'first-dir', 'test-backend-plugin'), + ); + writeFileSync( + join( + backstageRootDirectory, + 'first-dir', + 'test-backend-plugin', + 'package.json', + ), + JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + ); + + await waitForExpect(() => { + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(2); + }); + rootDirectorySubscriber.mockClear(); + await waitForExpect(() => { + expect(scannedPlugins).toEqual([ + { + location: new URL( + `file://${backstageRootDirectory}/first-dir/test-backend-plugin`, + ), + manifest: { + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + }, + }, + ]); + }); + + expect(logger.logs).toEqual({ + infos: [ + { + message: `rootDirectory changed (addDir - ${backstageRootDirectory}/first-dir/test-backend-plugin): scanning plugins again`, + }, + { + message: `rootDirectory changed (add - ${backstageRootDirectory}/first-dir/test-backend-plugin/package.json): scanning plugins again`, + }, + ], + }); + logger.logs = {}; + + getOptional.mockReturnValue({ + rootDirectory: 'second-dir', + }); + await onConfigChange!(); + + await waitForExpect(() => { + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(1); + }); + rootDirectorySubscriber.mockClear(); + await waitForExpect(() => { + expect(scannedPlugins).toEqual([]); + }); + + expect(logger.logs).toEqual({ + infos: [ + { + message: `rootDirectory changed in Config from '${backstageRootDirectory}/first-dir' to '${backstageRootDirectory}/second-dir'`, + }, + ], + }); + logger.logs = {}; + + mkdirSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + ), + ); + writeFileSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'package.json', + ), + JSON.stringify({ + name: 'second-test-backend-plugin-dynamic', + version: '1.0.3', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + ); + + await waitForExpect(() => { + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(2); + }); + rootDirectorySubscriber.mockClear(); + await waitForExpect(() => { + expect(scannedPlugins).toEqual([ + { + location: new URL( + `file://${backstageRootDirectory}/second-dir/second-test-backend-plugin`, + ), + manifest: { + backstage: { + role: 'backend-plugin', + }, + main: 'dist/index.cjs.js', + name: 'second-test-backend-plugin-dynamic', + version: '1.0.3', + }, + }, + ]); + }); + + expect(logger.logs).toEqual({ + infos: [ + { + message: `rootDirectory changed (addDir - ${backstageRootDirectory}/second-dir/second-test-backend-plugin): scanning plugins again`, + }, + { + message: `rootDirectory changed (add - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/package.json): scanning plugins again`, + }, + ], + }); + logger.logs = {}; + + const debug = jest.spyOn(logger, 'debug'); + + // Check that not all file changes trigger a new scan of plugins + mkdirSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'sub-directory', + ), + ); + await waitForExpect(() => { + expect(debug).toHaveBeenCalledTimes(1); + }); + writeFileSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'not-package.json', + ), + 'content', + ); + await waitForExpect(() => { + expect(debug).toHaveBeenCalledTimes(2); + }); + rmSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'not-package.json', + ), + ); + await waitForExpect(() => { + expect(debug).toHaveBeenCalledTimes(3); + }); + rmSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'sub-directory', + ), + { recursive: true }, + ); + await waitForExpect(() => { + expect(debug).toHaveBeenCalledTimes(4); + }); + + expect(logger.logs).toEqual({ + debugs: [ + { + message: `rootDirectory changed (addDir - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/sub-directory): no need to scan plugins again`, + }, + { + message: `rootDirectory changed (add - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/not-package.json): no need to scan plugins again`, + }, + { + message: `rootDirectory changed (unlink - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/not-package.json): no need to scan plugins again`, + }, + { + message: `rootDirectory changed (unlinkDir - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/sub-directory): no need to scan plugins again`, + }, + ], + }); + logger.logs = {}; + + // Now check that removal of some plugin home directory triggers a new scan of plugins + + rmSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + 'package.json', + ), + ); + await waitForExpect(() => { + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(1); + }); + rootDirectorySubscriber.mockClear(); + await waitForExpect(() => { + expect(scannedPlugins).toEqual([]); + }); + + expect(logger.logs).toEqual({ + infos: [ + { + message: `rootDirectory changed (unlink - ${backstageRootDirectory}/second-dir/second-test-backend-plugin/package.json): scanning plugins again`, + }, + ], + errors: [ + { + message: `failed to load dynamic plugin manifest from '${backstageRootDirectory}/second-dir/second-test-backend-plugin'`, + meta: { + code: 'ENOENT', + errno: -2, + message: `ENOENT: no such file or directory, open '${backstageRootDirectory}/second-dir/second-test-backend-plugin/package.json'`, + name: 'Error', + path: `${backstageRootDirectory}/second-dir/second-test-backend-plugin/package.json`, + syscall: 'open', + }, + }, + ], + }); + logger.logs = {}; + + rmSync( + join( + backstageRootDirectory, + 'second-dir', + 'second-test-backend-plugin', + ), + { recursive: true }, + ); + await waitForExpect(() => { + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(1); + }); + rootDirectorySubscriber.mockClear(); + + expect(logger.logs).toEqual({ + infos: [ + { + message: `rootDirectory changed (unlinkDir - ${backstageRootDirectory}/second-dir/second-test-backend-plugin): scanning plugins again`, + }, + ], + }); + logger.logs = {}; + + getOptional.mockReturnValue({ + rootDirectory: '/somewhere-else/second-dir', + }); + await onConfigChange!(); + + expect(logger.logs).toEqual({ + errors: [ + { + message: 'failed to apply new config for dynamic plugins', + meta: { + message: `Dynamic plugins under '/somewhere-else/second-dir' cannot access backstage modules in '${backstageRootDirectory}/node_modules'. +Please add '${backstageRootDirectory}/node_modules' to the 'NODE_PATH' when running the backstage backend.`, + name: 'Error', + }, + }, + ], + }); + }, 120000); + }); +}); diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts new file mode 100644 index 0000000000..aff671808b --- /dev/null +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts @@ -0,0 +1,683 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginScanner } from './plugin-scanner'; +import mockFs from 'mock-fs'; +import { JsonObject } from '@backstage/types'; +import { Logs, MockedLogger } from '../__testUtils__/testUtils'; +import { ConfigReader } from '@backstage/config'; +import path from 'path'; +import { ScannedPluginPackage } from './types'; + +describe('plugin-scanner', () => { + const env = process.env; + beforeEach(() => { + jest.resetModules(); + }); + + afterEach(() => { + mockFs.restore(); + process.env = env; + }); + + describe('applyConfig', () => { + type TestCase = { + name: string; + backstageRoot?: string; + fileSystem?: any; + config: JsonObject; + environment?: { [key: string]: string }; + expectedLogs?: Logs; + expectedRootDirectory?: string; + expectedError?: string; + }; + + it.each([ + { + name: 'no dynamic plugins', + config: {}, + expectedRootDirectory: undefined, + expectedLogs: { + infos: [ + { + message: "'dynamicPlugins' config entry not found.", + }, + ], + }, + }, + { + name: 'valid config with relative root directory path', + backstageRoot: '/backstageRoot', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory(), + }, + }), + }, + config: { + dynamicPlugins: { + rootDirectory: 'dist-dynamic', + }, + }, + expectedRootDirectory: '/backstageRoot/dist-dynamic', + }, + { + name: 'valid config with absolute root directory path inside the backstage root', + backstageRoot: '/backstageRoot', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory(), + }, + }), + }, + config: { + dynamicPlugins: { + rootDirectory: '/backstageRoot/dist-dynamic', + }, + }, + expectedRootDirectory: '/backstageRoot/dist-dynamic', + }, + { + name: 'valid config with absolute root directory path outside the backstage root', + backstageRoot: '/backstageRoot', + fileSystem: { + '/somewhere': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory(), + }, + }), + }, + config: { + dynamicPlugins: { + rootDirectory: '/somewhere/dist-dynamic', + }, + }, + expectedError: `Dynamic plugins under '/somewhere/dist-dynamic' cannot access backstage modules in '/backstageRoot/node_modules'. +Please add '/backstageRoot/node_modules' to the 'NODE_PATH' when running the backstage backend.`, + }, + { + name: 'valid config with absolute root directory path outside the backstage root but with backstage root included in NODE_PATH', + backstageRoot: '/backstageRoot', + fileSystem: { + '/somewhere': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory(), + }, + }), + }, + config: { + dynamicPlugins: { + rootDirectory: '/somewhere/dist-dynamic', + }, + }, + environment: { + NODE_PATH: `/somewhere-else${path.delimiter}/backstageRoot/node_modules${path.delimiter}/anywhere-else`, + }, + expectedRootDirectory: '/somewhere/dist-dynamic', + }, + { + name: 'invalid config: dynamicPlugins not an object', + config: { + dynamicPlugins: 1, + }, + expectedLogs: { + warns: [ + { + message: "'dynamicPlugins' config entry should be an object.", + }, + ], + }, + }, + { + name: 'invalid config: rootDirectory not a string', + config: { + dynamicPlugins: { + rootDirectory: 1, + }, + }, + expectedLogs: { + warns: [ + { + message: + "'dynamicPlugins.rootDirectory' config entry should be a string.", + }, + ], + }, + }, + { + name: 'invalid config: no rootDirectory entry', + config: { + dynamicPlugins: {}, + }, + expectedLogs: { + warns: [ + { + message: + "'dynamicPlugins' config entry does not contain the 'rootDirectory' field.", + }, + ], + }, + }, + { + name: 'valid config pointing to a file instead of a directory', + backstageRoot: '/backstageRoot', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.file(), + }, + }), + }, + config: { + dynamicPlugins: { + rootDirectory: 'dist-dynamic', + }, + }, + expectedError: 'Not a directory', + }, + ])('$name', (tc: TestCase): void => { + if (tc.environment) { + process.env = { + ...env, + ...tc.environment, + }; + } + const logger = new MockedLogger(); + const backstageRoot = tc.backstageRoot ? tc.backstageRoot : ''; + function toTest(): PluginScanner { + return new PluginScanner( + new ConfigReader(tc.config), + logger, + backstageRoot, + ); + } + if (tc.fileSystem) { + mockFs(tc.fileSystem); + } + if (tc.expectedError) { + /* eslint-disable-next-line jest/no-conditional-expect */ + expect(toTest).toThrow(tc.expectedError); + return; + } + const pluginScanner = toTest(); + if (tc.expectedLogs) { + /* eslint-disable-next-line jest/no-conditional-expect */ + expect(logger.logs).toEqual(tc.expectedLogs); + } else { + /* eslint-disable-next-line jest/no-conditional-expect */ + expect(logger.logs).toEqual({}); + } + expect(pluginScanner.rootDirectory).toEqual(tc.expectedRootDirectory); + }); + }); + describe('scanRoot', () => { + type TestCase = { + name: string; + preferAlpha?: boolean; + fileSystem?: any; + expectedLogs?: Logs; + expectedPluginPackages?: ScannedPluginPackage[]; + expectedError?: string; + }; + + it.each([ + { + name: 'dynamic plugins disabled', + expectedPluginPackages: [], + expectedLogs: { + infos: [ + { + message: "'dynamicPlugins' config entry not found.", + }, + ], + }, + }, + { + name: 'manifest found in directory', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [ + { + location: new URL( + 'file:///backstageRoot/dist-dynamic/test-backend-plugin', + ), + manifest: { + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }, + }, + ], + }, + { + name: 'backend plugin found in symlink', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.symlink({ + path: '/somewhere-else/test-backend-plugin-target', + }), + }, + }), + }, + }), + '/somewhere-else': mockFs.directory({ + items: { + 'test-backend-plugin-target': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [ + { + location: new URL( + 'file:///backstageRoot/dist-dynamic/test-backend-plugin', + ), + manifest: { + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }, + }, + ], + }, + { + name: 'ignored folder child: not a directory', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.file({}), + }, + }), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + infos: [ + { + message: + "skipping '/backstageRoot/dist-dynamic/test-backend-plugin' since it is not a directory", + }, + ], + }, + }, + { + name: 'ignored folder child symlink: target is not a directory', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.symlink({ + path: '/somewhere-else/test-backend-plugin-target', + }), + }, + }), + }, + }), + '/somewhere-else': mockFs.directory({ + items: { + 'test-backend-plugin-target': mockFs.file({}), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + infos: [ + { + message: + "skipping '/backstageRoot/dist-dynamic/test-backend-plugin' since it is not a directory", + }, + ], + }, + }, + { + name: 'alpha manifest available but not preferred', + preferAlpha: false, + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + alpha: mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: '../dist/alpha.cjs.js', + }), + }), + }, + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [ + { + location: new URL( + 'file:///backstageRoot/dist-dynamic/test-backend-plugin', + ), + manifest: { + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }, + }, + ], + }, + { + name: 'alpha manifest preferred and found in directory', + preferAlpha: true, + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + alpha: mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: '../dist/alpha.cjs.js', + }), + }), + }, + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [ + { + location: new URL( + 'file:///backstageRoot/dist-dynamic/test-backend-plugin/alpha', + ), + manifest: { + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: '../dist/alpha.cjs.js', + backstage: { role: 'backend-plugin' }, + }, + }, + ], + }, + { + name: 'alpha manifest preferred but skipped because not a directory', + preferAlpha: true, + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + alpha: mockFs.file({}), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [ + { + location: new URL( + 'file:///backstageRoot/dist-dynamic/test-backend-plugin', + ), + manifest: { + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }, + }, + ], + expectedLogs: { + warns: [ + { + message: + "skipping '/backstageRoot/dist-dynamic/test-backend-plugin/alpha' since it is not a directory", + }, + ], + }, + }, + { + name: 'invalid alpha package.json', + preferAlpha: true, + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + backstage: { role: 'backend-plugin' }, + }), + }), + alpha: mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: "invalid json content, 1, '", + }), + }, + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + errors: [ + { + message: + "failed to load dynamic plugin manifest from '/backstageRoot/dist-dynamic/test-backend-plugin/alpha'", + meta: { + name: 'SyntaxError', + message: 'Unexpected token i in JSON at position 0', + }, + }, + ], + }, + }, + { + name: 'invalid package.json', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: "invalid json content, 1, '", + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + errors: [ + { + message: + "failed to load dynamic plugin manifest from '/backstageRoot/dist-dynamic/test-backend-plugin'", + meta: { + name: 'SyntaxError', + message: 'Unexpected token i in JSON at position 0', + }, + }, + ], + }, + }, + { + name: 'missing backstage role in package.json', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + main: 'dist/index.cjs.js', + }), + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + errors: [ + { + message: + "failed to load dynamic plugin manifest from '/backstageRoot/dist-dynamic/test-backend-plugin'", + meta: { + name: 'TypeError', + message: "Cannot read properties of undefined (reading 'role')", + }, + }, + ], + }, + }, + ])('$name', async (tc: TestCase): Promise => { + const logger = new MockedLogger(); + const backstageRoot = '/backstageRoot'; + async function toTest(): Promise { + const pluginScanner = new PluginScanner( + new ConfigReader( + tc.fileSystem + ? { + dynamicPlugins: { + rootDirectory: 'dist-dynamic', + }, + } + : {}, + ), + logger, + backstageRoot, + tc.preferAlpha === undefined ? false : tc.preferAlpha, + ); + return await pluginScanner.scanRoot(); + } + if (tc.fileSystem) { + mockFs(tc.fileSystem); + } + if (tc.expectedError) { + /* eslint-disable-next-line jest/no-conditional-expect */ + expect(toTest).toThrow(tc.expectedError); + return; + } + const plugins = await toTest(); + expect(logger.logs).toEqual(tc.expectedLogs ? tc.expectedLogs : {}); + expect(plugins).toEqual(tc.expectedPluginPackages); + }); + }); +}); diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts new file mode 100644 index 0000000000..201f8cae8e --- /dev/null +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts @@ -0,0 +1,302 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config } from '@backstage/config'; +import { ScannedPluginPackage, ScannedPluginManifest } from './types'; +import * as fs from 'fs/promises'; +import { Stats, lstatSync } from 'fs'; +import * as chokidar from 'chokidar'; +import * as path from 'path'; +import * as url from 'url'; +import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +export class PluginScanner { + private readonly logger: LoggerService; + private backstageRoot: string; + readonly #config: Config; + private _rootDirectory?: string; + private readonly preferAlpha: boolean; + private configUnsubscribe?: () => void; + private rootDirectoryWatcher?: chokidar.FSWatcher; + private subscribers: (() => void)[] = []; + + constructor( + config: Config, + logger: LoggerService, + backstageRoot: string, + preferAlpha: boolean = false, + ) { + this.backstageRoot = backstageRoot; + this.logger = logger; + this.preferAlpha = preferAlpha; + this.#config = config; + + this.applyConfig(); + } + + subscribeToRootDirectoryChange(subscriber: () => void) { + this.subscribers.push(subscriber); + } + + get rootDirectory(): string | undefined { + return this._rootDirectory; + } + + private applyConfig(): void | never { + const dynamicPlugins = this.#config.getOptional('dynamicPlugins'); + if (!dynamicPlugins) { + this.logger.info("'dynamicPlugins' config entry not found."); + this._rootDirectory = undefined; + return; + } + if (typeof dynamicPlugins !== 'object') { + this.logger.warn("'dynamicPlugins' config entry should be an object."); + this._rootDirectory = undefined; + return; + } + if (!('rootDirectory' in dynamicPlugins)) { + this.logger.warn( + "'dynamicPlugins' config entry does not contain the 'rootDirectory' field.", + ); + this._rootDirectory = undefined; + return; + } + if (typeof dynamicPlugins.rootDirectory !== 'string') { + this.logger.warn( + "'dynamicPlugins.rootDirectory' config entry should be a string.", + ); + this._rootDirectory = undefined; + return; + } + + const dynamicPluginsRootPath = path.isAbsolute(dynamicPlugins.rootDirectory) + ? dynamicPlugins.rootDirectory + : path.resolve(this.backstageRoot, dynamicPlugins.rootDirectory); + + if ( + !path + .dirname(path.normalize(dynamicPluginsRootPath)) + .startsWith(path.normalize(this.backstageRoot)) + ) { + const nodePath = process.env.NODE_PATH; + const backstageNodeModules = path.resolve( + this.backstageRoot, + 'node_modules', + ); + if ( + !nodePath || + !nodePath.split(path.delimiter).includes(backstageNodeModules) + ) { + throw new Error( + `Dynamic plugins under '${dynamicPluginsRootPath}' cannot access backstage modules in '${backstageNodeModules}'.\n` + + `Please add '${backstageNodeModules}' to the 'NODE_PATH' when running the backstage backend.`, + ); + } + } + if (!lstatSync(dynamicPluginsRootPath).isDirectory()) { + throw new Error('Not a directory'); + } + + this._rootDirectory = dynamicPluginsRootPath; + } + + async scanRoot(): Promise { + if (!this._rootDirectory) { + return []; + } + + const dynamicPluginsLocation = this._rootDirectory; + const scannedPlugins: ScannedPluginPackage[] = []; + for (const dirEnt of await fs.readdir(dynamicPluginsLocation, { + withFileTypes: true, + })) { + const pluginDir = dirEnt; + const pluginHome = path.normalize( + path.resolve(dynamicPluginsLocation, pluginDir.name), + ); + if (dirEnt.isSymbolicLink()) { + if (!(await fs.lstat(await fs.readlink(pluginHome))).isDirectory()) { + this.logger.info( + `skipping '${pluginHome}' since it is not a directory`, + ); + continue; + } + } else if (!dirEnt.isDirectory()) { + this.logger.info( + `skipping '${pluginHome}' since it is not a directory`, + ); + continue; + } + + let scannedPlugin: ScannedPluginPackage; + try { + scannedPlugin = await this.scanDir(pluginHome); + } catch (e) { + this.logger.error( + `failed to load dynamic plugin manifest from '${pluginHome}'`, + e, + ); + continue; + } + + let platform: PackagePlatform; + try { + platform = PackageRoles.getRoleInfo( + scannedPlugin.manifest.backstage.role, + ).platform; + } catch (e) { + this.logger.error( + `failed to load dynamic plugin manifest from '${pluginHome}'`, + e, + ); + continue; + } + + if (platform === 'node') { + if (this.preferAlpha) { + const pluginHomeAlpha = path.resolve(pluginHome, 'alpha'); + if ((await fs.lstat(pluginHomeAlpha)).isDirectory()) { + const backstage = scannedPlugin.manifest.backstage; + try { + scannedPlugin = await this.scanDir(pluginHomeAlpha); + } catch (e) { + this.logger.error( + `failed to load dynamic plugin manifest from '${pluginHomeAlpha}'`, + e, + ); + continue; + } + scannedPlugin.manifest.backstage = backstage; + } else { + this.logger.warn( + `skipping '${pluginHomeAlpha}' since it is not a directory`, + ); + } + } + } + + scannedPlugins.push(scannedPlugin); + } + return scannedPlugins; + } + + private async scanDir(pluginHome: string): Promise { + const manifestFile = path.resolve(pluginHome, 'package.json'); + const content = await fs.readFile(manifestFile); + const manifest: ScannedPluginManifest = JSON.parse(content.toString()); + return { + location: url.pathToFileURL(pluginHome), + manifest: manifest, + }; + } + + async trackChanges(): Promise { + const setupRootDirectoryWatcher = async (): Promise => { + return new Promise((resolve, reject) => { + if (!this._rootDirectory) { + resolve(); + return; + } + let ready = false; + this.rootDirectoryWatcher = chokidar + .watch(this._rootDirectory, { + ignoreInitial: true, + followSymlinks: true, + }) + .on( + 'all', + ( + event: 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir', + eventPath: string, + _: Stats | undefined, + ): void => { + if ( + (['addDir', 'unlinkDir'].includes(event) && + path.dirname(eventPath) === this._rootDirectory) || + (['add', 'unlink', 'change'].includes(event) && + path.dirname(path.dirname(eventPath)) === + this._rootDirectory && + path.basename(eventPath) === 'package.json') + ) { + this.logger.info( + `rootDirectory changed (${event} - ${eventPath}): scanning plugins again`, + ); + this.subscribers.forEach(s => s()); + } else { + this.logger.debug( + `rootDirectory changed (${event} - ${eventPath}): no need to scan plugins again`, + ); + } + }, + ) + .on('error', (error: Error) => { + this.logger.error( + `error while watching '${this.rootDirectory}'`, + error, + ); + if (!ready) { + reject(error); + } + }) + .on('ready', () => { + ready = true; + resolve(); + }); + }); + }; + + await setupRootDirectoryWatcher(); + if (this.#config.subscribe) { + const { unsubscribe } = this.#config.subscribe( + async (): Promise => { + const oldRootDirectory = this._rootDirectory; + try { + this.applyConfig(); + } catch (e) { + this.logger.error( + 'failed to apply new config for dynamic plugins', + e, + ); + } + if (oldRootDirectory !== this._rootDirectory) { + this.logger.info( + `rootDirectory changed in Config from '${oldRootDirectory}' to '${this._rootDirectory}'`, + ); + this.subscribers.forEach(s => s()); + if (this.rootDirectoryWatcher) { + await this.rootDirectoryWatcher.close(); + } + await setupRootDirectoryWatcher(); + } + }, + ); + this.configUnsubscribe = unsubscribe; + } + } + + async untrackChanges() { + if (this.rootDirectoryWatcher) { + this.rootDirectoryWatcher.close(); + } + if (this.configUnsubscribe) { + this.configUnsubscribe(); + } + } + + destructor() { + this.untrackChanges(); + } +} diff --git a/packages/backend-plugin-manager/src/scanner/types.ts b/packages/backend-plugin-manager/src/scanner/types.ts new file mode 100644 index 0000000000..726babaaaa --- /dev/null +++ b/packages/backend-plugin-manager/src/scanner/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PackageRole } from '@backstage/cli-node'; + +/** + * @public + */ +export interface ScannedPluginPackage { + location: URL; + manifest: ScannedPluginManifest; +} + +/** + * @public + */ +export interface ScannedPluginManifest { + name: string; + version: string; + backstage: { + role: PackageRole; + }; + main: string; +} diff --git a/yarn.lock b/yarn.lock index 82c9012702..a2ca6a4f76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3470,6 +3470,40 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-plugin-manager@workspace:packages/backend-plugin-manager": + version: 0.0.0-use.local + resolution: "@backstage/backend-plugin-manager@workspace:packages/backend-plugin-manager" + dependencies: + "@backstage/backend-app-api": "workspace:^" + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/cli-common": "workspace:^" + "@backstage/cli-node": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/config-loader": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/plugin-auth-node": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-events-backend": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@backstage/plugin-permission-node": "workspace:^" + "@backstage/plugin-scaffolder-node": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" + "@backstage/plugin-search-common": "workspace:^" + "@backstage/types": "workspace:^" + "@types/express": ^4.17.6 + chokidar: ^3.5.3 + express: ^4.17.1 + mock-fs: ^5.2.0 + wait-for-expect: ^3.0.2 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/backend-tasks@workspace:^, @backstage/backend-tasks@workspace:packages/backend-tasks": version: 0.0.0-use.local resolution: "@backstage/backend-tasks@workspace:packages/backend-tasks" From 23e4a9f6b64dcb79517d56c01b21d9be70229102 Mon Sep 17 00:00:00 2001 From: David Festal Date: Wed, 9 Aug 2023 17:41:22 +0200 Subject: [PATCH 2/6] Add changeset Signed-off-by: David Festal --- .changeset/happy-falcons-relax.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/happy-falcons-relax.md diff --git a/.changeset/happy-falcons-relax.md b/.changeset/happy-falcons-relax.md new file mode 100644 index 0000000000..c1cc98e2f3 --- /dev/null +++ b/.changeset/happy-falcons-relax.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-plugin-manager': minor +--- + +New package with experimental support for dynamic backend plugins, according to [RFC 18390](https://github.com/backstage/backstage/issues/18390). From 66849e0677fd274be9e502b7a5a0bb9b86a7f0a3 Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 28 Aug 2023 17:52:06 +0200 Subject: [PATCH 3/6] Fix review comments Signed-off-by: David Festal --- packages/backend-plugin-manager/api-report.md | 42 +++++++++---------- packages/backend-plugin-manager/config.d.ts | 2 - packages/backend-plugin-manager/package.json | 4 +- .../src/__testUtils__/testUtils.ts | 12 +++--- packages/backend-plugin-manager/src/index.ts | 7 ++-- .../src/loader/index.ts | 17 ++++++++ .../src/manager/index.ts | 38 +++++++++++++++++ .../src/manager/plugin-manager.test.ts | 14 +++---- .../src/manager/plugin-manager.ts | 6 +-- .../src/manager/types.ts | 23 ++++++---- .../src/scanner/index.ts | 17 ++++++++ .../scanner/plugin-scanner-watcher.test.ts | 38 ++++++++++------- .../src/scanner/plugin-scanner.ts | 6 ++- yarn.lock | 1 + 14 files changed, 158 insertions(+), 69 deletions(-) create mode 100644 packages/backend-plugin-manager/src/loader/index.ts create mode 100644 packages/backend-plugin-manager/src/manager/index.ts create mode 100644 packages/backend-plugin-manager/src/scanner/index.ts diff --git a/packages/backend-plugin-manager/api-report.md b/packages/backend-plugin-manager/api-report.md index e10f71ba15..026b1d0553 100644 --- a/packages/backend-plugin-manager/api-report.md +++ b/packages/backend-plugin-manager/api-report.md @@ -101,11 +101,11 @@ export function isBackendDynamicPluginInstaller( // @public @deprecated (undocumented) export interface LegacyBackendPluginInstaller { // (undocumented) - catalog?(builder: CatalogBuilder, env: PluginEnvironment): void; + catalog?(builder: CatalogBuilder, env: LegacyPluginEnvironment): void; // (undocumented) events?( eventsBackend: EventsBackend, - env: PluginEnvironment, + env: LegacyPluginEnvironment, ): HttpPostIngressOptions[]; // (undocumented) kind: 'legacy'; @@ -116,18 +116,34 @@ export interface LegacyBackendPluginInstaller { // (undocumented) router?: { pluginID: string; - createPlugin(env: PluginEnvironment): Promise; + createPlugin(env: LegacyPluginEnvironment): Promise; }; // (undocumented) - scaffolder?(env: PluginEnvironment): TemplateAction[]; + scaffolder?(env: LegacyPluginEnvironment): TemplateAction[]; // (undocumented) search?( indexBuilder: IndexBuilder, schedule: TaskRunner, - env: PluginEnvironment, + env: LegacyPluginEnvironment, ): void; } +// @public @deprecated (undocumented) +export type LegacyPluginEnvironment = { + logger: Logger; + cache: PluginCacheManager; + database: PluginDatabaseManager; + config: Config; + reader: UrlReader; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + permissions: PermissionEvaluator; + scheduler: PluginTaskScheduler; + identity: IdentityApi; + eventBroker: EventBroker; + pluginProvider: BackendPluginProvider; +}; + // @public (undocumented) export interface ModuleLoader { // (undocumented) @@ -146,22 +162,6 @@ export interface NewBackendPluginInstaller { kind: 'new'; } -// @public (undocumented) -export type PluginEnvironment = { - logger: Logger; - cache: PluginCacheManager; - database: PluginDatabaseManager; - config: Config; - reader: UrlReader; - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - permissions: PermissionEvaluator; - scheduler: PluginTaskScheduler; - identity: IdentityApi; - eventBroker: EventBroker; - pluginProvider: BackendPluginProvider; -}; - // @public (undocumented) export class PluginManager implements BackendPluginProvider { // (undocumented) diff --git a/packages/backend-plugin-manager/config.d.ts b/packages/backend-plugin-manager/config.d.ts index edbc99ce89..9427458a23 100644 --- a/packages/backend-plugin-manager/config.d.ts +++ b/packages/backend-plugin-manager/config.d.ts @@ -15,11 +15,9 @@ */ export interface Config { - /** @visibility frontend */ dynamicPlugins?: { /** * The local path, relative to the backstage root, that contains dynamic plugins. - * @visibility frontend */ rootDirectory: string; }; diff --git a/packages/backend-plugin-manager/package.json b/packages/backend-plugin-manager/package.json index 12bfd179fb..de9fb8fa63 100644 --- a/packages/backend-plugin-manager/package.json +++ b/packages/backend-plugin-manager/package.json @@ -1,8 +1,7 @@ { "name": "@backstage/backend-plugin-manager", - "packageManager": "yarn@3.2.3", "description": "Backstage plugin management backend", - "version": "0.1.0", + "version": "0.0.0", "main": "src/index.ts", "types": "src/index.ts", "publishConfig": { @@ -47,6 +46,7 @@ "@types/express": "^4.17.6", "chokidar": "^3.5.3", "express": "^4.17.1", + "lodash": "^4.17.21", "winston": "^3.2.1" }, "devDependencies": { diff --git a/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts b/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts index 57bd9e4973..11a4421239 100644 --- a/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts +++ b/packages/backend-plugin-manager/src/__testUtils__/testUtils.ts @@ -17,13 +17,13 @@ import { LoggerService } from '@backstage/backend-plugin-api'; import { JsonObject } from '@backstage/types'; -export type logContent = { message: string; meta?: Error | JsonObject }; +export type LogContent = { message: string; meta?: Error | JsonObject }; export type Logs = { - errors?: logContent[]; - warns?: logContent[]; - infos?: logContent[]; - debugs?: logContent[]; + errors?: LogContent[]; + warns?: LogContent[]; + infos?: LogContent[]; + debugs?: LogContent[]; }; export class MockedLogger implements LoggerService { @@ -59,7 +59,7 @@ export class MockedLogger implements LoggerService { } } -function toLogContent(message: string, meta?: Error | JsonObject): logContent { +function toLogContent(message: string, meta?: Error | JsonObject): LogContent { if (!meta) { return { message }; } diff --git a/packages/backend-plugin-manager/src/index.ts b/packages/backend-plugin-manager/src/index.ts index 997913454e..5570d8ee29 100644 --- a/packages/backend-plugin-manager/src/index.ts +++ b/packages/backend-plugin-manager/src/index.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -export * from './scanner/types'; -export * from './loader/types'; -export * from './manager/types'; -export * from './manager/plugin-manager'; +export * from './loader'; +export * from './scanner'; +export * from './manager'; diff --git a/packages/backend-plugin-manager/src/loader/index.ts b/packages/backend-plugin-manager/src/loader/index.ts new file mode 100644 index 0000000000..ff0892e66d --- /dev/null +++ b/packages/backend-plugin-manager/src/loader/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { ModuleLoader } from './types'; diff --git a/packages/backend-plugin-manager/src/manager/index.ts b/packages/backend-plugin-manager/src/manager/index.ts new file mode 100644 index 0000000000..a9cd163617 --- /dev/null +++ b/packages/backend-plugin-manager/src/manager/index.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { isBackendDynamicPluginInstaller } from './types'; + +export type { + BaseDynamicPlugin, + DynamicPlugin, + FrontendDynamicPlugin, + BackendDynamicPlugin, + BackendDynamicPluginInstaller, + NewBackendPluginInstaller, + LegacyBackendPluginInstaller, + LegacyPluginEnvironment, + BackendPluginProvider, +} from './types'; + +export { + PluginManager, + dynamicPluginsFeatureDiscoveryServiceFactory, + dynamicPluginsServiceFactory, + dynamicPluginsServiceRef, +} from './plugin-manager'; + +export type { DynamicPluginsFactoryOptions } from './plugin-manager'; diff --git a/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts index 2d0966abbe..9d086092b9 100644 --- a/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts +++ b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts @@ -30,7 +30,7 @@ import { DynamicPlugin, LegacyBackendPluginInstaller, NewBackendPluginInstaller, - PluginEnvironment, + LegacyPluginEnvironment, } from './types'; import { ScannedPluginManifest, ScannedPluginPackage } from '../scanner/types'; import { randomUUID } from 'crypto'; @@ -40,7 +40,7 @@ import { rootLifecycleServiceFactory, } from '@backstage/backend-app-api'; import { ConfigSources } from '@backstage/config-loader'; -import { Logs, MockedLogger, logContent } from '../__testUtils__/testUtils'; +import { Logs, MockedLogger, LogContent } from '../__testUtils__/testUtils'; import { PluginScanner } from '../scanner/plugin-scanner'; import { findPaths } from '@backstage/cli-common'; @@ -59,10 +59,10 @@ describe('backend-plugin-manager', () => { content?: string; }; expectedLogs?(location: URL): { - errors?: logContent[]; - warns?: logContent[]; - infos?: logContent[]; - debugs?: logContent[]; + errors?: LogContent[]; + warns?: LogContent[]; + infos?: LogContent[]; + debugs?: LogContent[]; }; checkLoadedPlugins: (loadedPlugins: BaseDynamicPlugin[]) => void; }; @@ -327,7 +327,7 @@ describe('backend-plugin-manager', () => { ]); const installer = (plugins[0] as BackendDynamicPlugin) .installer as LegacyBackendPluginInstaller; - expect(installer.scaffolder!({} as PluginEnvironment)).toEqual< + expect(installer.scaffolder!({} as LegacyPluginEnvironment)).toEqual< TemplateAction[] >([]); }, diff --git a/packages/backend-plugin-manager/src/manager/plugin-manager.ts b/packages/backend-plugin-manager/src/manager/plugin-manager.ts index b4cfaeeb64..89ea144426 100644 --- a/packages/backend-plugin-manager/src/manager/plugin-manager.ts +++ b/packages/backend-plugin-manager/src/manager/plugin-manager.ts @@ -20,9 +20,11 @@ import { isBackendDynamicPluginInstaller, DynamicPlugin, } from './types'; +import { ScannedPluginPackage } from '../scanner'; import { PluginScanner } from '../scanner/plugin-scanner'; +import { ModuleLoader } from '../loader'; +import { CommonJSModuleLoader } from '../loader/CommonJSModuleLoader'; import * as url from 'url'; -import { ScannedPluginPackage } from '../scanner/types'; import { BackendFeature, LoggerService, @@ -34,8 +36,6 @@ import { PackageRoles } from '@backstage/cli-node'; import { findPaths } from '@backstage/cli-common'; import path from 'path'; import * as fs from 'fs'; -import { ModuleLoader } from '../loader/types'; -import { CommonJSModuleLoader } from '../loader/CommonJSModuleLoader'; import { FeatureDiscoveryService, featureDiscoveryServiceRef, diff --git a/packages/backend-plugin-manager/src/manager/types.ts b/packages/backend-plugin-manager/src/manager/types.ts index 8c220b7699..201941292d 100644 --- a/packages/backend-plugin-manager/src/manager/types.ts +++ b/packages/backend-plugin-manager/src/manager/types.ts @@ -42,8 +42,17 @@ import { PermissionPolicy } from '@backstage/plugin-permission-node'; /** * @public + * + * @deprecated + * + * Support for the legacy backend system will be removed in the future. + * + * When adding a legacy plugin installer entrypoint in your plugin, + * you should always take the opportunity to also implement support + * for the new backend system if not already done. + * */ -export type PluginEnvironment = { +export type LegacyPluginEnvironment = { logger: Logger; cache: PluginCacheManager; database: PluginDatabaseManager; @@ -115,7 +124,7 @@ export interface NewBackendPluginInstaller { * @public * @deprecated * - * Support for the legacy backend suystem will be removed in the future. + * Support for the legacy backend system will be removed in the future. * * When adding a legacy plugin installer entrypoint in your plugin, * you should always take the opportunity to also implement support @@ -127,19 +136,19 @@ export interface LegacyBackendPluginInstaller { router?: { pluginID: string; - createPlugin(env: PluginEnvironment): Promise; + createPlugin(env: LegacyPluginEnvironment): Promise; }; - catalog?(builder: CatalogBuilder, env: PluginEnvironment): void; - scaffolder?(env: PluginEnvironment): TemplateAction[]; + catalog?(builder: CatalogBuilder, env: LegacyPluginEnvironment): void; + scaffolder?(env: LegacyPluginEnvironment): TemplateAction[]; search?( indexBuilder: IndexBuilder, schedule: TaskRunner, - env: PluginEnvironment, + env: LegacyPluginEnvironment, ): void; events?( eventsBackend: EventsBackend, - env: PluginEnvironment, + env: LegacyPluginEnvironment, ): HttpPostIngressOptions[]; permissions?: { policy?: PermissionPolicy; diff --git a/packages/backend-plugin-manager/src/scanner/index.ts b/packages/backend-plugin-manager/src/scanner/index.ts new file mode 100644 index 0000000000..c5574cc38e --- /dev/null +++ b/packages/backend-plugin-manager/src/scanner/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { ScannedPluginManifest, ScannedPluginPackage } from './types'; diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts index 2644024cd7..5ced3cfbab 100644 --- a/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner-watcher.test.ts @@ -19,7 +19,8 @@ import { Logs, MockedLogger } from '../__testUtils__/testUtils'; import { Config, ConfigReader } from '@backstage/config'; import path, { join } from 'path'; import { ScannedPluginPackage } from './types'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { mkdtempSync, rmSync } from 'fs'; +import { mkdir, writeFile, rm } from 'fs/promises'; import { tmpdir } from 'os'; import waitForExpect from 'wait-for-expect'; @@ -33,15 +34,20 @@ describe('plugin-scanner', () => { }); afterEach(() => { if (backstageRootDirectory) { - rmSync(backstageRootDirectory, { recursive: true }); + rmSync(backstageRootDirectory, { recursive: true, force: true }); + } + }); + process.on('beforeExit', () => { + if (backstageRootDirectory) { + rmSync(backstageRootDirectory, { recursive: true, force: true }); } }); it('watch for config and file system changes', async () => { const logger = new MockedLogger(); - mkdirSync(join(backstageRootDirectory, 'first-dir')); - mkdirSync(join(backstageRootDirectory, 'second-dir')); + await mkdir(join(backstageRootDirectory, 'first-dir')); + await mkdir(join(backstageRootDirectory, 'second-dir')); const config: Config = new ConfigReader({ dynamicPlugins: { @@ -77,10 +83,10 @@ describe('plugin-scanner', () => { }); pluginScanner.subscribeToRootDirectoryChange(rootDirectorySubscriber); - mkdirSync( + await mkdir( join(backstageRootDirectory, 'first-dir', 'test-backend-plugin'), ); - writeFileSync( + await writeFile( join( backstageRootDirectory, 'first-dir', @@ -96,7 +102,7 @@ describe('plugin-scanner', () => { ); await waitForExpect(() => { - expect(rootDirectorySubscriber).toHaveBeenCalledTimes(2); + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(1); }); rootDirectorySubscriber.mockClear(); await waitForExpect(() => { @@ -151,14 +157,14 @@ describe('plugin-scanner', () => { }); logger.logs = {}; - mkdirSync( + await mkdir( join( backstageRootDirectory, 'second-dir', 'second-test-backend-plugin', ), ); - writeFileSync( + await writeFile( join( backstageRootDirectory, 'second-dir', @@ -174,7 +180,7 @@ describe('plugin-scanner', () => { ); await waitForExpect(() => { - expect(rootDirectorySubscriber).toHaveBeenCalledTimes(2); + expect(rootDirectorySubscriber).toHaveBeenCalledTimes(1); }); rootDirectorySubscriber.mockClear(); await waitForExpect(() => { @@ -210,7 +216,7 @@ describe('plugin-scanner', () => { const debug = jest.spyOn(logger, 'debug'); // Check that not all file changes trigger a new scan of plugins - mkdirSync( + await mkdir( join( backstageRootDirectory, 'second-dir', @@ -221,7 +227,7 @@ describe('plugin-scanner', () => { await waitForExpect(() => { expect(debug).toHaveBeenCalledTimes(1); }); - writeFileSync( + await writeFile( join( backstageRootDirectory, 'second-dir', @@ -233,7 +239,7 @@ describe('plugin-scanner', () => { await waitForExpect(() => { expect(debug).toHaveBeenCalledTimes(2); }); - rmSync( + await rm( join( backstageRootDirectory, 'second-dir', @@ -244,7 +250,7 @@ describe('plugin-scanner', () => { await waitForExpect(() => { expect(debug).toHaveBeenCalledTimes(3); }); - rmSync( + await rm( join( backstageRootDirectory, 'second-dir', @@ -277,7 +283,7 @@ describe('plugin-scanner', () => { // Now check that removal of some plugin home directory triggers a new scan of plugins - rmSync( + await rm( join( backstageRootDirectory, 'second-dir', @@ -315,7 +321,7 @@ describe('plugin-scanner', () => { }); logger.logs = {}; - rmSync( + await rm( join( backstageRootDirectory, 'second-dir', diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts index 201f8cae8e..39d55bdb5c 100644 --- a/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts @@ -20,6 +20,7 @@ import { Stats, lstatSync } from 'fs'; import * as chokidar from 'chokidar'; import * as path from 'path'; import * as url from 'url'; +import debounce from 'lodash/debounce'; import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -210,6 +211,9 @@ export class PluginScanner { resolve(); return; } + const callSubscribers = debounce(() => { + this.subscribers.forEach(s => s()); + }, 500); let ready = false; this.rootDirectoryWatcher = chokidar .watch(this._rootDirectory, { @@ -234,7 +238,7 @@ export class PluginScanner { this.logger.info( `rootDirectory changed (${event} - ${eventPath}): scanning plugins again`, ); - this.subscribers.forEach(s => s()); + callSubscribers(); } else { this.logger.debug( `rootDirectory changed (${event} - ${eventPath}): no need to scan plugins again`, diff --git a/yarn.lock b/yarn.lock index a2ca6a4f76..6d6bfca66f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3498,6 +3498,7 @@ __metadata: "@types/express": ^4.17.6 chokidar: ^3.5.3 express: ^4.17.1 + lodash: ^4.17.21 mock-fs: ^5.2.0 wait-for-expect: ^3.0.2 winston: ^3.2.1 From 3bd7516c8ee2e9322d4326e9ca9107cbf7ad3fb7 Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 28 Aug 2023 21:56:57 +0200 Subject: [PATCH 4/6] Fixes after rebase to master Signed-off-by: David Festal --- packages/backend-plugin-manager/README.md | 16 ++++------------ .../src/manager/plugin-manager.test.ts | 11 +++++++++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/packages/backend-plugin-manager/README.md b/packages/backend-plugin-manager/README.md index 42b68739a9..c9fd666fb3 100644 --- a/packages/backend-plugin-manager/README.md +++ b/packages/backend-plugin-manager/README.md @@ -17,19 +17,11 @@ The backend plugin manager is a service that scans a configured root directory ( In the `backend-next` application, it can be enabled by adding the `backend-plugin-manager` as a dependency in the `package.json` and the following lines in the `src/index.ts` file: ```ts -- --const backend = createBackend(); -+import { -+ dynamicPluginsServiceFactory, -+ dynamicPluginsFeatureDiscoveryServiceFactory, -+} from '@backstage/backend-plugin-manager'; +const backend = createBackend(); ++ ++ backend.add(dynamicPluginsFeatureDiscoveryServiceFactory()) // overridden version of the FeatureDiscoveryService which provides features loaded by dynamic plugins ++ backend.add(dynamicPluginsServiceFactory()) + -+const backend = createBackend({ -+ services: [ -+ dynamicPluginsServiceFactory(), -+ dynamicPluginsFeatureDiscoveryServiceFactory(), // overridden version of the FeatureDiscoveryService which provides features loaded by dynamic plugins -+ ], -+}); ``` ### About the expected dynamic plugin structure diff --git a/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts index 9d086092b9..8c54538b5e 100644 --- a/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts +++ b/packages/backend-plugin-manager/src/manager/plugin-manager.test.ts @@ -486,7 +486,7 @@ describe('backend-plugin-manager', () => { }; const backend = createSpecializedBackend({ - services: [ + defaultServiceFactories: [ rootLifecycleServiceFactory(), createServiceFactory({ service: coreServices.rootConfig, @@ -494,7 +494,14 @@ describe('backend-plugin-manager', () => { async factory({}) { return await ConfigSources.toConfig({ async *readConfigData() { - yield { configs: [] }; + yield { + configs: [ + { + context: 'test', + data: {}, + }, + ], + }; }, }); }, From 27a139dad54e882f5c4334767a0e9958f24ad678 Mon Sep 17 00:00:00 2001 From: David Festal Date: Thu, 31 Aug 2023 11:53:24 +0200 Subject: [PATCH 5/6] Fixes after second round of review Signed-off-by: David Festal --- packages/backend-plugin-manager/api-report.md | 18 +++----- .../src/loader/types.ts | 4 -- .../src/scanner/plugin-scanner.test.ts | 41 ++++++++++++++++++- .../src/scanner/plugin-scanner.ts | 24 +++++------ .../src/scanner/types.ts | 13 +++--- 5 files changed, 60 insertions(+), 40 deletions(-) diff --git a/packages/backend-plugin-manager/api-report.md b/packages/backend-plugin-manager/api-report.md index 026b1d0553..9617dc1096 100644 --- a/packages/backend-plugin-manager/api-report.md +++ b/packages/backend-plugin-manager/api-report.md @@ -4,6 +4,7 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { BackstagePackageJson } from '@backstage/cli-node'; import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; import { EventBroker } from '@backstage/plugin-events-node'; @@ -150,8 +151,6 @@ export interface ModuleLoader { bootstrap(backstageRoot: string, dynamicPluginPaths: string[]): Promise; // (undocumented) load(id: string): Promise; - // (undocumented) - logger: LoggerService; } // @public (undocumented) @@ -182,18 +181,11 @@ export class PluginManager implements BackendPluginProvider { } // @public (undocumented) -export interface ScannedPluginManifest { - // (undocumented) - backstage: { - role: PackageRole; +export type ScannedPluginManifest = BackstagePackageJson & + Required> & + Required> & { + backstage: Required; }; - // (undocumented) - main: string; - // (undocumented) - name: string; - // (undocumented) - version: string; -} // @public (undocumented) export interface ScannedPluginPackage { diff --git a/packages/backend-plugin-manager/src/loader/types.ts b/packages/backend-plugin-manager/src/loader/types.ts index ebd3a1e7ce..12a051e482 100644 --- a/packages/backend-plugin-manager/src/loader/types.ts +++ b/packages/backend-plugin-manager/src/loader/types.ts @@ -14,14 +14,10 @@ * limitations under the License. */ -import { LoggerService } from '@backstage/backend-plugin-api'; - /** * @public */ export interface ModuleLoader { - logger: LoggerService; - bootstrap(backstageRoot: string, dynamicPluginPaths: string[]): Promise; load(id: string): Promise; diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts index aff671808b..25f07baeaa 100644 --- a/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner.test.ts @@ -640,8 +640,45 @@ Please add '/backstageRoot/node_modules' to the 'NODE_PATH' when running the bac message: "failed to load dynamic plugin manifest from '/backstageRoot/dist-dynamic/test-backend-plugin'", meta: { - name: 'TypeError', - message: "Cannot read properties of undefined (reading 'role')", + name: 'Error', + message: "field 'backstage.role' not found in 'package.json'", + }, + }, + ], + }, + }, + { + name: 'missing main field in package.json', + fileSystem: { + '/backstageRoot': mockFs.directory({ + items: { + 'dist-dynamic': mockFs.directory({ + items: { + 'test-backend-plugin': mockFs.directory({ + items: { + 'package.json': mockFs.file({ + content: JSON.stringify({ + name: 'test-backend-plugin-dynamic', + version: '0.0.0', + backstage: { role: 'backend-plugin' }, + }), + }), + }, + }), + }, + }), + }, + }), + }, + expectedPluginPackages: [], + expectedLogs: { + errors: [ + { + message: + "failed to load dynamic plugin manifest from '/backstageRoot/dist-dynamic/test-backend-plugin'", + meta: { + name: 'Error', + message: "field 'main' not found in 'package.json'", }, }, ], diff --git a/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts index 39d55bdb5c..bdcf786619 100644 --- a/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts +++ b/packages/backend-plugin-manager/src/scanner/plugin-scanner.ts @@ -143,21 +143,19 @@ export class PluginScanner { } let scannedPlugin: ScannedPluginPackage; - try { - scannedPlugin = await this.scanDir(pluginHome); - } catch (e) { - this.logger.error( - `failed to load dynamic plugin manifest from '${pluginHome}'`, - e, - ); - continue; - } - let platform: PackagePlatform; try { - platform = PackageRoles.getRoleInfo( - scannedPlugin.manifest.backstage.role, - ).platform; + scannedPlugin = await this.scanDir(pluginHome); + if (!scannedPlugin.manifest.main) { + throw new Error("field 'main' not found in 'package.json'"); + } + if (scannedPlugin.manifest.backstage?.role) { + platform = PackageRoles.getRoleInfo( + scannedPlugin.manifest.backstage.role, + ).platform; + } else { + throw new Error("field 'backstage.role' not found in 'package.json'"); + } } catch (e) { this.logger.error( `failed to load dynamic plugin manifest from '${pluginHome}'`, diff --git a/packages/backend-plugin-manager/src/scanner/types.ts b/packages/backend-plugin-manager/src/scanner/types.ts index 726babaaaa..b456ffeea1 100644 --- a/packages/backend-plugin-manager/src/scanner/types.ts +++ b/packages/backend-plugin-manager/src/scanner/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PackageRole } from '@backstage/cli-node'; +import { BackstagePackageJson } from '@backstage/cli-node'; /** * @public @@ -27,11 +27,8 @@ export interface ScannedPluginPackage { /** * @public */ -export interface ScannedPluginManifest { - name: string; - version: string; - backstage: { - role: PackageRole; +export type ScannedPluginManifest = BackstagePackageJson & + Required> & + Required> & { + backstage: Required; }; - main: string; -} From 4cfa636a323f1ac4cdd96f2b08fc29964bbbbafa Mon Sep 17 00:00:00 2001 From: David Festal Date: Thu, 31 Aug 2023 15:52:15 +0200 Subject: [PATCH 6/6] Make the package `private` Signed-off-by: David Festal --- .changeset/happy-falcons-relax.md | 5 ----- packages/backend-plugin-manager/package.json | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 .changeset/happy-falcons-relax.md diff --git a/.changeset/happy-falcons-relax.md b/.changeset/happy-falcons-relax.md deleted file mode 100644 index c1cc98e2f3..0000000000 --- a/.changeset/happy-falcons-relax.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-plugin-manager': minor ---- - -New package with experimental support for dynamic backend plugins, according to [RFC 18390](https://github.com/backstage/backstage/issues/18390). diff --git a/packages/backend-plugin-manager/package.json b/packages/backend-plugin-manager/package.json index de9fb8fa63..2549675ce9 100644 --- a/packages/backend-plugin-manager/package.json +++ b/packages/backend-plugin-manager/package.json @@ -2,6 +2,7 @@ "name": "@backstage/backend-plugin-manager", "description": "Backstage plugin management backend", "version": "0.0.0", + "private": true, "main": "src/index.ts", "types": "src/index.ts", "publishConfig": {