diff --git a/.changeset/swift-pumpkins-shake.md b/.changeset/swift-pumpkins-shake.md new file mode 100644 index 0000000000..c9441a3690 --- /dev/null +++ b/.changeset/swift-pumpkins-shake.md @@ -0,0 +1,12 @@ +--- +'@backstage/backend-app-api': minor +'@backstage/backend-common': minor +'@backstage/config-loader': minor +'@backstage/plugin-app-backend': minor +'@backstage/backend-dynamic-feature-service': minor +--- + +Allow referencing additional configuration schemas at runtime (application start), and have them taken in account where schemas are used: + +- the backed-app plugin, in order to prepare the frontend configuration +- the backend application loggers, in order to hide secret configuration values (defined in the schemas). diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index e478a91bb7..1143beec15 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -9,6 +9,7 @@ import type { AppConfig } from '@backstage/config'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CacheClient } from '@backstage/backend-common'; import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { CorsOptions } from 'cors'; import { DiscoveryService } from '@backstage/backend-plugin-api'; import { ErrorRequestHandler } from 'express'; @@ -64,6 +65,7 @@ export const cacheServiceFactory: () => ServiceFactory; export function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; + additionalSchemas?: ConfigSchemaPackageEntry[]; }): Promise<(config: Config) => Iterable>; // @public diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index 8ecc015b66..d4eb5fccb9 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -23,6 +23,7 @@ import { loadConfig, ConfigTarget, LoadConfigOptionsRemote, + ConfigSchemaPackageEntry, } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import type { Config, AppConfig } from '@backstage/config'; @@ -34,11 +35,13 @@ import { isValidUrl } from '../lib/urls'; export async function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; + additionalSchemas?: ConfigSchemaPackageEntry[]; }): Promise<(config: Config) => Iterable> { const { logger, dir = process.cwd() } = options; const { packages } = await getPackages(dir); const schema = await loadConfigSchema({ dependencies: packages.map(p => p.packageJson.name), + additionalSchemas: options.additionalSchemas, }); return (config: Config) => { diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts new file mode 100644 index 0000000000..ac7b9e8cf2 --- /dev/null +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createSpecializedBackend } from '../../../wiring'; +import { rootLoggerServiceFactory } from './rootLoggerServiceFactory'; +import { + coreServices, + createBackendPlugin, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; +import { + ConfigSchemaPackageEntry, + ConfigSources, + StaticConfigSource, +} from '@backstage/config-loader'; +import { transports } from 'winston'; +import { rootLifecycleServiceFactory } from '../rootLifecycle'; +import { lifecycleServiceFactory } from '../lifecycle'; +import { loggerServiceFactory } from '../logger'; + +describe('rootLogger', () => { + describe('rootLoggerServiceFactory', () => { + afterEach(() => { + jest.resetModules(); + }); + + it('also hide secret config values coming from additional schemas', async () => { + const additionalConfigs = { + context: 'test', + data: { + secretValue: 'shouldBeHidden', + }, + }; + + const additionalSchemas = [ + { + path: 'test', + value: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', + }, + }, + }, + }, + ]; + + const logs: string[] = []; + jest + .spyOn(transports.Console.prototype, 'log') + .mockImplementation((s: any, next: any) => { + logs.push(s.message); + if (next) { + next(); + } + return undefined; + }); + + const backend = createSpecializedBackend({ + defaultServiceFactories: [ + lifecycleServiceFactory(), + loggerServiceFactory(), + rootLifecycleServiceFactory(), + rootLoggerServiceFactory(), + createServiceFactory({ + service: coreServices.rootConfig, + deps: {}, + async factory() { + return await ConfigSources.toConfig( + ConfigSources.merge([ + ConfigSources.default({}), + StaticConfigSource.create(additionalConfigs), + ]), + ); + }, + }), + createServiceFactory({ + service: schemaDiscoveryServiceRef, + deps: { + config: coreServices.rootConfig, + }, + factory: async () => ({ + getAdditionalSchemas: async (): Promise<{ + schemas: Array; + }> => ({ + schemas: additionalSchemas, + }), + }), + }), + ], + }); + backend.add( + createBackendPlugin({ + pluginId: 'test', + register(env) { + env.registerInit({ + deps: { + logger: coreServices.logger, + lifecycle: coreServices.lifecycle, + }, + async init({ logger }) { + logger.info('test'); + logger.info('shouldBeHidden'); + }, + }); + }, + }), + ); + await backend.start(); + + expect(logs[0]).toMatch( + /Found \d+ new secrets in config that will be redacted/, + ); + expect(logs[1]).toEqual('test'); + expect(logs[2]).toEqual('[REDACTED]'); + }, 30000); + }); +}); diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts index bd45df383f..becf618f4f 100644 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts @@ -18,6 +18,7 @@ import { createServiceFactory, coreServices, } from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { WinstonLogger } from '../../../logging'; import { transports, format } from 'winston'; import { createConfigSecretEnumerator } from '../../../config'; @@ -27,8 +28,9 @@ export const rootLoggerServiceFactory = createServiceFactory({ service: coreServices.rootLogger, deps: { config: coreServices.rootConfig, + schemaDiscovery: schemaDiscoveryServiceRef, }, - async factory({ config }) { + async factory({ config, schemaDiscovery }) { const logger = WinstonLogger.create({ meta: { service: 'backstage', @@ -41,7 +43,12 @@ export const rootLoggerServiceFactory = createServiceFactory({ transports: [new transports.Console()], }); - const secretEnumerator = await createConfigSecretEnumerator({ logger }); + const secretEnumerator = await createConfigSecretEnumerator({ + logger, + additionalSchemas: ( + await schemaDiscovery?.getAdditionalSchemas() + )?.schemas, + }); logger.addRedactions(secretEnumerator(config)); config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index e4a1ecfdf1..7a3fc72a5b 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -20,6 +20,7 @@ import { CacheService as CacheClient } from '@backstage/backend-plugin-api'; import { CacheServiceOptions as CacheClientOptions } from '@backstage/backend-plugin-api'; import { CacheServiceSetOptions as CacheClientSetOptions } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import cors from 'cors'; import Docker from 'dockerode'; import { ErrorRequestHandler } from 'express'; @@ -560,6 +561,7 @@ export function loadBackendConfig(options: { logger: LoggerService; remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; + additionalSchemas?: ConfigSchemaPackageEntry[]; argv: string[]; watch?: boolean; }): Promise; diff --git a/packages/backend-common/src/config.test.ts b/packages/backend-common/src/config.test.ts new file mode 100644 index 0000000000..931c0b5c84 --- /dev/null +++ b/packages/backend-common/src/config.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { transports } from 'winston'; +import { loadBackendConfig } from './config'; +import { getRootLogger } from './logging'; + +describe('config', () => { + describe('loadBackendConfig', () => { + const env = process.env; + afterEach(() => { + jest.resetModules(); + process.env = env; + }); + + it('also hide secret config values coming from additional schemas', async () => { + const additionalConfigs = [ + { + context: 'test', + data: { + secretValue: 'shouldBeHidden', + }, + }, + ]; + + const additionalSchemas = [ + { + path: 'test', + value: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', + }, + }, + }, + }, + ]; + + const logs: string[] = []; + jest + .spyOn(transports.Console.prototype, 'log') + .mockImplementation((s: any, next: any) => { + logs.push(s.message); + if (next) { + next(); + } + return undefined; + }); + + process.env.LOG_LEVEL = 'info'; + const logger = getRootLogger(); + await loadBackendConfig({ + logger, + argv: [], + additionalConfigs, + additionalSchemas, + }); + logger.info('test'); + logger.info('shouldBeHidden'); + logger.end(); + await new Promise(resolve => { + logger.on('finish', resolve); + }); + + expect(logs[0]).toMatch( + /Found \d+ new secrets in config that will be redacted/, + ); + expect(logs[1]).toEqual('test'); + expect(logs[2]).toEqual('[REDACTED]'); + }, 30000); + }); +}); diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index b452d8db70..b5d1d2b896 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,10 @@ import { } from '@backstage/backend-app-api'; import { LoggerService } from '@backstage/backend-plugin-api'; import { AppConfig, Config } from '@backstage/config'; -import { LoadConfigOptionsRemote } from '@backstage/config-loader'; +import { + ConfigSchemaPackageEntry, + LoadConfigOptionsRemote, +} from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; /** @@ -35,11 +38,13 @@ export async function loadBackendConfig(options: { // process.argv or any other overrides remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; + additionalSchemas?: ConfigSchemaPackageEntry[]; argv: string[]; watch?: boolean; }): Promise { const secretEnumerator = await createConfigSecretEnumerator({ logger: options.logger, + additionalSchemas: options.additionalSchemas, }); const { config } = await newLoadBackendConfig(options); diff --git a/packages/backend-dynamic-feature-service/api-report.md b/packages/backend-dynamic-feature-service/api-report.md index caf2cf01ec..55a710dbe0 100644 --- a/packages/backend-dynamic-feature-service/api-report.md +++ b/packages/backend-dynamic-feature-service/api-report.md @@ -24,6 +24,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Router } from 'express'; +import { SchemaDiscoveryService } from '@backstage/backend-plugin-api/alpha'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -115,6 +116,11 @@ export const dynamicPluginsFeatureDiscoveryServiceFactory: () => ServiceFactory< 'root' >; +// @public (undocumented) +export interface DynamicPluginsSchemaDiscoveryOptions { + schemaLocator?: (platform: PackagePlatform) => string; +} + // @public (undocumented) export const dynamicPluginsServiceFactory: ( options?: DynamicPluginsFactoryOptions | undefined, @@ -220,5 +226,10 @@ export interface ScannedPluginPackage { manifest: ScannedPluginManifest; } +// @public (undocumented) +export const schemaDiscoveryServiceFactory: ( + options?: DynamicPluginsSchemaDiscoveryOptions | undefined, +) => ServiceFactory; + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-dynamic-feature-service/package.json b/packages/backend-dynamic-feature-service/package.json index 5a3b88eda6..bc99c2a692 100644 --- a/packages/backend-dynamic-feature-service/package.json +++ b/packages/backend-dynamic-feature-service/package.json @@ -32,6 +32,7 @@ "@backstage/cli-common": "workspace:^", "@backstage/cli-node": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", @@ -46,6 +47,7 @@ "@types/express": "^4.17.6", "chokidar": "^3.5.3", "express": "^4.17.1", + "fs-extra": "^7.0.1", "lodash": "^4.17.21", "winston": "^3.2.1" }, @@ -53,7 +55,6 @@ "@backstage/backend-app-api": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@backstage/config-loader": "workspace:^", "wait-for-expect": "^3.0.2" }, "files": [ diff --git a/packages/backend-dynamic-feature-service/src/scanner/index.ts b/packages/backend-dynamic-feature-service/src/scanner/index.ts index c5574cc38e..22b476c143 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/index.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/index.ts @@ -15,3 +15,5 @@ */ export type { ScannedPluginManifest, ScannedPluginPackage } from './types'; +export type { DynamicPluginsSchemaDiscoveryOptions } from './plugin-scanner'; +export { schemaDiscoveryServiceFactory } from './plugin-scanner'; diff --git a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts index af1144f019..7f8996503c 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts @@ -22,7 +22,15 @@ import * as path from 'path'; import * as url from 'url'; import debounce from 'lodash/debounce'; import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + LoggerService, + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import { findPaths } from '@backstage/cli-common'; +import { gatherDynamicPluginsSchemas } from './schemas'; export interface DynamicPluginScannerOptions { config: Config; @@ -315,3 +323,70 @@ export class PluginScanner { this.untrackChanges(); } } + +/** + * @public + */ +export interface DynamicPluginsSchemaDiscoveryOptions { + /** + * Function that returns the plugin-relative path to the Json schema file for a given platform. + * Default behavior is to look for the `configSchema.json` file in the package `dist` sub-directory. + * + * @param platform - The platform of the plugin. + * @returns the plugin-relative path to the Json schema file. + */ + schemaLocator?: (platform: PackagePlatform) => string; +} + +/** + * @public + */ +export const schemaDiscoveryServiceFactory = createServiceFactory( + (options?: DynamicPluginsSchemaDiscoveryOptions) => ({ + service: schemaDiscoveryServiceRef, + deps: { + config: coreServices.rootConfig, + }, + factory({ config }) { + let schemas: ConfigSchemaPackageEntry[] | undefined; + + return { + async getAdditionalSchemas(): Promise<{ + schemas: Array; + }> { + if (schemas) { + return { + schemas, + }; + } + + const logger = { + ...console, + child() { + return this; + }, + }; + + const scanner = PluginScanner.create({ + config, + logger, + // eslint-disable-next-line no-restricted-syntax + backstageRoot: findPaths(__dirname).targetRoot, + preferAlpha: true, + }); + + const { packages } = await scanner.scanRoot(); + + schemas = await gatherDynamicPluginsSchemas( + packages, + logger, + options?.schemaLocator, + ); + return { + schemas, + }; + }, + }; + }, + }), +); diff --git a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts new file mode 100644 index 0000000000..b6d55181a9 --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScannedPluginPackage } from '@backstage/backend-dynamic-feature-service'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import fs from 'fs-extra'; +import * as path from 'path'; +import * as url from 'url'; +import { isEmpty } from 'lodash'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; + +export async function gatherDynamicPluginsSchemas( + packages: ScannedPluginPackage[], + logger: LoggerService, + schemaLocator: (platform: PackagePlatform) => string = () => + path.join('dist', 'configSchema.json'), +): Promise { + const allSchemas: { value: any; path: string }[] = []; + + for (const pluginPackage of packages) { + const platform = PackageRoles.getRoleInfo( + pluginPackage.manifest.backstage.role, + ).platform; + + let pluginLocation = url.fileURLToPath(pluginPackage.location); + if (path.basename(pluginLocation) === 'alpha') { + pluginLocation = path.dirname(pluginLocation); + } + const schemaLocation: string = path.resolve( + pluginLocation, + schemaLocator(platform), + ); + + if (!(await fs.pathExists(schemaLocation))) { + continue; + } + + const serialized = await fs.readJson(schemaLocation); + if (!serialized) { + continue; + } + + if (isEmpty(serialized)) { + continue; + } + + if (!serialized?.$schema || serialized?.type !== 'object') { + logger.error( + `Serialized configuration schema is invalid for plugin ${pluginPackage.manifest.name}`, + ); + continue; + } + + allSchemas.push({ + path: schemaLocation, + value: serialized, + }); + } + + return allSchemas; +} diff --git a/packages/backend-plugin-api/api-report-alpha.md b/packages/backend-plugin-api/api-report-alpha.md index 81b378a671..33b5e748e7 100644 --- a/packages/backend-plugin-api/api-report-alpha.md +++ b/packages/backend-plugin-api/api-report-alpha.md @@ -4,6 +4,7 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -20,5 +21,19 @@ export const featureDiscoveryServiceRef: ServiceRef< 'root' >; +// @alpha (undocumented) +export interface SchemaDiscoveryService { + // (undocumented) + getAdditionalSchemas(): Promise<{ + schemas: Array; + }>; +} + +// @alpha +export const schemaDiscoveryServiceRef: ServiceRef< + SchemaDiscoveryService, + 'root' +>; + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-plugin-api/package.json b/packages/backend-plugin-api/package.json index a2946bd803..5476a75cf3 100644 --- a/packages/backend-plugin-api/package.json +++ b/packages/backend-plugin-api/package.json @@ -47,6 +47,7 @@ "dependencies": { "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/types": "workspace:^", diff --git a/packages/backend-plugin-api/src/alpha.ts b/packages/backend-plugin-api/src/alpha.ts index baee739f49..e5eac99e4a 100644 --- a/packages/backend-plugin-api/src/alpha.ts +++ b/packages/backend-plugin-api/src/alpha.ts @@ -16,8 +16,11 @@ import { BackendFeature, + coreServices, + createServiceFactory, createServiceRef, } from '@backstage/backend-plugin-api'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; /** @alpha */ export interface FeatureDiscoveryService { @@ -33,3 +36,32 @@ export const featureDiscoveryServiceRef = id: 'core.featureDiscovery', scope: 'root', }); + +/** @alpha */ +export interface SchemaDiscoveryService { + getAdditionalSchemas(): Promise<{ schemas: Array }>; +} + +/** + * An optional service that can be used to dynamically load in additional BackendFeatures at runtime. + * @alpha + */ +export const schemaDiscoveryServiceRef = + createServiceRef({ + id: 'core.schemaDiscovery', + scope: 'root', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + config: coreServices.rootConfig, + }, + factory() { + return { + async getAdditionalSchemas() { + return { schemas: [] }; + }, + }; + }, + }), + }); diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 66572dd960..04f1b6a77b 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -45,6 +45,12 @@ export type ConfigSchema = { serialize(): JsonObject; }; +// @public +export type ConfigSchemaPackageEntry = { + value: JsonObject; + path: string; +}; + // @public export type ConfigSchemaProcessingOptions = { visibility?: ConfigVisibility[]; @@ -193,6 +199,7 @@ export type LoadConfigSchemaOptions = ( } ) & { noUndeclaredProperties?: boolean; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; // @public diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index e20e105c7f..80435d6000 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -27,6 +27,7 @@ export type { ConfigVisibility, LoadConfigSchemaOptions, TransformFunc, + ConfigSchemaPackageEntry, } from './schema'; export { loadConfig } from './loader'; export type { diff --git a/packages/config-loader/src/schema/index.ts b/packages/config-loader/src/schema/index.ts index 1dcb9d7b4b..01c18ee905 100644 --- a/packages/config-loader/src/schema/index.ts +++ b/packages/config-loader/src/schema/index.ts @@ -22,4 +22,5 @@ export type { ConfigVisibility, ConfigSchemaProcessingOptions, TransformFunc, + ConfigSchemaPackageEntry, } from './types'; diff --git a/packages/config-loader/src/schema/load.test.ts b/packages/config-loader/src/schema/load.test.ts index 525565f3d5..107f8d1f3b 100644 --- a/packages/config-loader/src/schema/load.test.ts +++ b/packages/config-loader/src/schema/load.test.ts @@ -126,6 +126,60 @@ describe('loadConfigSchema', () => { ); }); + it('should include additional schemas', async () => { + const schema = await loadConfigSchema({ + serialized: { + backstageConfigSchemaVersion: 1, + schemas: [ + { + path: 'mainSchema', + value: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }, + additionalSchemas: [ + { + path: 'additionalSchema', + value: { + type: 'object', + properties: { + additionalKey: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }); + + expect(schema.serialize()).toEqual({ + backstageConfigSchemaVersion: 1, + schemas: [ + { + path: 'mainSchema', + value: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + { + path: 'additionalSchema', + value: { + type: 'object', + properties: { + additionalKey: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }); + }); + describe('should consider schema', () => { it('when filtering simple config', async () => { mockDir.setContent({ diff --git a/packages/config-loader/src/schema/load.ts b/packages/config-loader/src/schema/load.ts index 885da44116..5c535d1ff6 100644 --- a/packages/config-loader/src/schema/load.ts +++ b/packages/config-loader/src/schema/load.ts @@ -43,6 +43,7 @@ export type LoadConfigSchemaOptions = } ) & { noUndeclaredProperties?: boolean; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; function errorsToError(errors: ValidationError[]): Error { @@ -83,6 +84,7 @@ export async function loadConfigSchema( } schemas = serialized.schemas as ConfigSchemaPackageEntry[]; } + schemas.push(...(options.additionalSchemas || [])); const validate = compileConfigSchemas(schemas, { noUndeclaredProperties: options.noUndeclaredProperties, diff --git a/packages/config-loader/src/schema/types.ts b/packages/config-loader/src/schema/types.ts index 8f679c93fc..66e77018ee 100644 --- a/packages/config-loader/src/schema/types.ts +++ b/packages/config-loader/src/schema/types.ts @@ -18,7 +18,9 @@ import { AppConfig } from '@backstage/config'; import { JsonObject } from '@backstage/types'; /** - * An sub-set of configuration schema. + * A sub-set of configuration schema for a given package. + * + * @public */ export type ConfigSchemaPackageEntry = { /** diff --git a/plugins/app-backend/api-report.md b/plugins/app-backend/api-report.md index 0383129823..9ea42d2fbd 100644 --- a/plugins/app-backend/api-report.md +++ b/plugins/app-backend/api-report.md @@ -4,6 +4,7 @@ ```ts import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import express from 'express'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -13,6 +14,7 @@ export function createRouter(options: RouterOptions): Promise; // @public (undocumented) export interface RouterOptions { + additionalSchemas?: ConfigSchemaPackageEntry[]; appPackageName: string; // (undocumented) config: Config; diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 63912df5c4..0e223c3650 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -20,6 +20,7 @@ import { Logger } from 'winston'; import { AppConfig, Config } from '@backstage/config'; import { JsonObject } from '@backstage/types'; import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; type InjectOptions = { appConfigs: AppConfig[]; @@ -74,6 +75,7 @@ type ReadOptions = { env: { [name: string]: string | undefined }; appDistDir: string; config: Config; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; /** @@ -90,7 +92,10 @@ export async function readConfigs(options: ReadOptions): Promise { const serializedSchema = await fs.readJson(schemaPath); try { - const schema = await loadConfigSchema({ serialized: serializedSchema }); + const schema = await loadConfigSchema({ + serialized: serializedSchema, + additionalSchemas: options.additionalSchemas, + }); const frontendConfigs = await schema.process( [{ data: config.get() as JsonObject, context: 'app' }], diff --git a/plugins/app-backend/src/service/appPlugin.ts b/plugins/app-backend/src/service/appPlugin.ts index 9ba2ea6d6c..07616e832f 100644 --- a/plugins/app-backend/src/service/appPlugin.ts +++ b/plugins/app-backend/src/service/appPlugin.ts @@ -19,6 +19,7 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { createRouter } from './router'; import { loggerToWinstonLogger } from '@backstage/backend-common'; import { staticFallbackHandlerExtensionPoint } from '@backstage/plugin-app-node'; @@ -49,8 +50,9 @@ export const appPlugin = createBackendPlugin({ config: coreServices.rootConfig, database: coreServices.database, httpRouter: coreServices.httpRouter, + schemaDiscovery: schemaDiscoveryServiceRef, }, - async init({ logger, config, database, httpRouter }) { + async init({ logger, config, database, httpRouter, schemaDiscovery }) { const appPackageName = config.getOptionalString('app.packageName') ?? 'app'; const winstonLogger = loggerToWinstonLogger(logger); @@ -61,6 +63,9 @@ export const appPlugin = createBackendPlugin({ database, appPackageName, staticFallbackHandler, + additionalSchemas: ( + await schemaDiscovery?.getAdditionalSchemas() + )?.schemas, }); httpRouter.use(router); }, diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index 5574892d37..8e4750dcef 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -37,6 +37,7 @@ import { CACHE_CONTROL_NO_CACHE, CACHE_CONTROL_REVALIDATE_CACHE, } from '../lib/headers'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; // express uses mime v1 while we only have types for mime v2 type Mime = { lookup(arg0: string): string }; @@ -85,6 +86,17 @@ export interface RouterOptions { * This also disables configuration injection though `APP_CONFIG_` environment variables. */ disableConfigInjection?: boolean; + + /** + * + * Provides a list of additional config schemas, in addition to the serialized schemas + * generated during the application build. + * This is useful when additional plugins are dynamically loaded in the application at start, + * which were not part of the application build. This option allows feeding the corresponding + * JSON schemas. + * + */ + additionalSchemas?: ConfigSchemaPackageEntry[]; } /** @public */ @@ -121,6 +133,7 @@ export async function createRouter( config, appDistDir, env: process.env, + additionalSchemas: options.additionalSchemas, }); injectedConfigPath = await injectConfig({ appConfigs, logger, staticDir }); diff --git a/yarn.lock b/yarn.lock index 5f84cbbc51..0e32adda8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3395,6 +3395,7 @@ __metadata: "@types/express": ^4.17.6 chokidar: ^3.5.3 express: ^4.17.1 + fs-extra: ^7.0.1 lodash: ^4.17.21 wait-for-expect: ^3.0.2 winston: ^3.2.1 @@ -3428,6 +3429,7 @@ __metadata: "@backstage/backend-tasks": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/config-loader": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/types": "workspace:^"