From d7adbbf455a7395ef37098d3c2fbef090905eeed Mon Sep 17 00:00:00 2001 From: David Festal Date: Wed, 29 Nov 2023 16:37:36 +0100 Subject: [PATCH 01/34] Support external config schemas in the backend. Signed-off-by: David Festal --- .changeset/swift-pumpkins-shake.md | 12 ++ packages/backend-app-api/api-report.md | 2 + packages/backend-app-api/src/config/config.ts | 3 + .../rootLoggerServiceFactory.test.ts | 134 ++++++++++++++++++ .../rootLogger/rootLoggerServiceFactory.ts | 11 +- packages/backend-common/api-report.md | 2 + packages/backend-common/src/config.test.ts | 87 ++++++++++++ packages/backend-common/src/config.ts | 7 +- .../api-report.md | 11 ++ .../package.json | 3 +- .../src/scanner/index.ts | 2 + .../src/scanner/plugin-scanner.ts | 77 +++++++++- .../src/scanner/schemas.ts | 75 ++++++++++ .../backend-plugin-api/api-report-alpha.md | 15 ++ packages/backend-plugin-api/package.json | 1 + packages/backend-plugin-api/src/alpha.ts | 32 +++++ packages/config-loader/api-report.md | 7 + packages/config-loader/src/index.ts | 1 + packages/config-loader/src/schema/index.ts | 1 + .../config-loader/src/schema/load.test.ts | 54 +++++++ packages/config-loader/src/schema/load.ts | 2 + packages/config-loader/src/schema/types.ts | 4 +- plugins/app-backend/api-report.md | 2 + plugins/app-backend/src/lib/config.ts | 7 +- plugins/app-backend/src/service/appPlugin.ts | 7 +- plugins/app-backend/src/service/router.ts | 13 ++ yarn.lock | 2 + 27 files changed, 566 insertions(+), 8 deletions(-) create mode 100644 .changeset/swift-pumpkins-shake.md create mode 100644 packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts create mode 100644 packages/backend-common/src/config.test.ts create mode 100644 packages/backend-dynamic-feature-service/src/scanner/schemas.ts diff --git a/.changeset/swift-pumpkins-shake.md b/.changeset/swift-pumpkins-shake.md new file mode 100644 index 0000000000..c9441a3690 --- /dev/null +++ b/.changeset/swift-pumpkins-shake.md @@ -0,0 +1,12 @@ +--- +'@backstage/backend-app-api': minor +'@backstage/backend-common': minor +'@backstage/config-loader': minor +'@backstage/plugin-app-backend': minor +'@backstage/backend-dynamic-feature-service': minor +--- + +Allow referencing additional configuration schemas at runtime (application start), and have them taken in account where schemas are used: + +- the backed-app plugin, in order to prepare the frontend configuration +- the backend application loggers, in order to hide secret configuration values (defined in the schemas). diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index e478a91bb7..1143beec15 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -9,6 +9,7 @@ import type { AppConfig } from '@backstage/config'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CacheClient } from '@backstage/backend-common'; import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { CorsOptions } from 'cors'; import { DiscoveryService } from '@backstage/backend-plugin-api'; import { ErrorRequestHandler } from 'express'; @@ -64,6 +65,7 @@ export const cacheServiceFactory: () => ServiceFactory; export function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; + additionalSchemas?: ConfigSchemaPackageEntry[]; }): Promise<(config: Config) => Iterable>; // @public diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index 8ecc015b66..d4eb5fccb9 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -23,6 +23,7 @@ import { loadConfig, ConfigTarget, LoadConfigOptionsRemote, + ConfigSchemaPackageEntry, } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import type { Config, AppConfig } from '@backstage/config'; @@ -34,11 +35,13 @@ import { isValidUrl } from '../lib/urls'; export async function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; + additionalSchemas?: ConfigSchemaPackageEntry[]; }): Promise<(config: Config) => Iterable> { const { logger, dir = process.cwd() } = options; const { packages } = await getPackages(dir); const schema = await loadConfigSchema({ dependencies: packages.map(p => p.packageJson.name), + additionalSchemas: options.additionalSchemas, }); return (config: Config) => { diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts new file mode 100644 index 0000000000..ac7b9e8cf2 --- /dev/null +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createSpecializedBackend } from '../../../wiring'; +import { rootLoggerServiceFactory } from './rootLoggerServiceFactory'; +import { + coreServices, + createBackendPlugin, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; +import { + ConfigSchemaPackageEntry, + ConfigSources, + StaticConfigSource, +} from '@backstage/config-loader'; +import { transports } from 'winston'; +import { rootLifecycleServiceFactory } from '../rootLifecycle'; +import { lifecycleServiceFactory } from '../lifecycle'; +import { loggerServiceFactory } from '../logger'; + +describe('rootLogger', () => { + describe('rootLoggerServiceFactory', () => { + afterEach(() => { + jest.resetModules(); + }); + + it('also hide secret config values coming from additional schemas', async () => { + const additionalConfigs = { + context: 'test', + data: { + secretValue: 'shouldBeHidden', + }, + }; + + const additionalSchemas = [ + { + path: 'test', + value: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', + }, + }, + }, + }, + ]; + + const logs: string[] = []; + jest + .spyOn(transports.Console.prototype, 'log') + .mockImplementation((s: any, next: any) => { + logs.push(s.message); + if (next) { + next(); + } + return undefined; + }); + + const backend = createSpecializedBackend({ + defaultServiceFactories: [ + lifecycleServiceFactory(), + loggerServiceFactory(), + rootLifecycleServiceFactory(), + rootLoggerServiceFactory(), + createServiceFactory({ + service: coreServices.rootConfig, + deps: {}, + async factory() { + return await ConfigSources.toConfig( + ConfigSources.merge([ + ConfigSources.default({}), + StaticConfigSource.create(additionalConfigs), + ]), + ); + }, + }), + createServiceFactory({ + service: schemaDiscoveryServiceRef, + deps: { + config: coreServices.rootConfig, + }, + factory: async () => ({ + getAdditionalSchemas: async (): Promise<{ + schemas: Array; + }> => ({ + schemas: additionalSchemas, + }), + }), + }), + ], + }); + backend.add( + createBackendPlugin({ + pluginId: 'test', + register(env) { + env.registerInit({ + deps: { + logger: coreServices.logger, + lifecycle: coreServices.lifecycle, + }, + async init({ logger }) { + logger.info('test'); + logger.info('shouldBeHidden'); + }, + }); + }, + }), + ); + await backend.start(); + + expect(logs[0]).toMatch( + /Found \d+ new secrets in config that will be redacted/, + ); + expect(logs[1]).toEqual('test'); + expect(logs[2]).toEqual('[REDACTED]'); + }, 30000); + }); +}); diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts index bd45df383f..becf618f4f 100644 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts @@ -18,6 +18,7 @@ import { createServiceFactory, coreServices, } from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { WinstonLogger } from '../../../logging'; import { transports, format } from 'winston'; import { createConfigSecretEnumerator } from '../../../config'; @@ -27,8 +28,9 @@ export const rootLoggerServiceFactory = createServiceFactory({ service: coreServices.rootLogger, deps: { config: coreServices.rootConfig, + schemaDiscovery: schemaDiscoveryServiceRef, }, - async factory({ config }) { + async factory({ config, schemaDiscovery }) { const logger = WinstonLogger.create({ meta: { service: 'backstage', @@ -41,7 +43,12 @@ export const rootLoggerServiceFactory = createServiceFactory({ transports: [new transports.Console()], }); - const secretEnumerator = await createConfigSecretEnumerator({ logger }); + const secretEnumerator = await createConfigSecretEnumerator({ + logger, + additionalSchemas: ( + await schemaDiscovery?.getAdditionalSchemas() + )?.schemas, + }); logger.addRedactions(secretEnumerator(config)); config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index e4a1ecfdf1..7a3fc72a5b 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -20,6 +20,7 @@ import { CacheService as CacheClient } from '@backstage/backend-plugin-api'; import { CacheServiceOptions as CacheClientOptions } from '@backstage/backend-plugin-api'; import { CacheServiceSetOptions as CacheClientSetOptions } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import cors from 'cors'; import Docker from 'dockerode'; import { ErrorRequestHandler } from 'express'; @@ -560,6 +561,7 @@ export function loadBackendConfig(options: { logger: LoggerService; remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; + additionalSchemas?: ConfigSchemaPackageEntry[]; argv: string[]; watch?: boolean; }): Promise; diff --git a/packages/backend-common/src/config.test.ts b/packages/backend-common/src/config.test.ts new file mode 100644 index 0000000000..931c0b5c84 --- /dev/null +++ b/packages/backend-common/src/config.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { transports } from 'winston'; +import { loadBackendConfig } from './config'; +import { getRootLogger } from './logging'; + +describe('config', () => { + describe('loadBackendConfig', () => { + const env = process.env; + afterEach(() => { + jest.resetModules(); + process.env = env; + }); + + it('also hide secret config values coming from additional schemas', async () => { + const additionalConfigs = [ + { + context: 'test', + data: { + secretValue: 'shouldBeHidden', + }, + }, + ]; + + const additionalSchemas = [ + { + path: 'test', + value: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', + }, + }, + }, + }, + ]; + + const logs: string[] = []; + jest + .spyOn(transports.Console.prototype, 'log') + .mockImplementation((s: any, next: any) => { + logs.push(s.message); + if (next) { + next(); + } + return undefined; + }); + + process.env.LOG_LEVEL = 'info'; + const logger = getRootLogger(); + await loadBackendConfig({ + logger, + argv: [], + additionalConfigs, + additionalSchemas, + }); + logger.info('test'); + logger.info('shouldBeHidden'); + logger.end(); + await new Promise(resolve => { + logger.on('finish', resolve); + }); + + expect(logs[0]).toMatch( + /Found \d+ new secrets in config that will be redacted/, + ); + expect(logs[1]).toEqual('test'); + expect(logs[2]).toEqual('[REDACTED]'); + }, 30000); + }); +}); diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index b452d8db70..b5d1d2b896 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,7 +20,10 @@ import { } from '@backstage/backend-app-api'; import { LoggerService } from '@backstage/backend-plugin-api'; import { AppConfig, Config } from '@backstage/config'; -import { LoadConfigOptionsRemote } from '@backstage/config-loader'; +import { + ConfigSchemaPackageEntry, + LoadConfigOptionsRemote, +} from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; /** @@ -35,11 +38,13 @@ export async function loadBackendConfig(options: { // process.argv or any other overrides remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; + additionalSchemas?: ConfigSchemaPackageEntry[]; argv: string[]; watch?: boolean; }): Promise { const secretEnumerator = await createConfigSecretEnumerator({ logger: options.logger, + additionalSchemas: options.additionalSchemas, }); const { config } = await newLoadBackendConfig(options); diff --git a/packages/backend-dynamic-feature-service/api-report.md b/packages/backend-dynamic-feature-service/api-report.md index caf2cf01ec..55a710dbe0 100644 --- a/packages/backend-dynamic-feature-service/api-report.md +++ b/packages/backend-dynamic-feature-service/api-report.md @@ -24,6 +24,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Router } from 'express'; +import { SchemaDiscoveryService } from '@backstage/backend-plugin-api/alpha'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -115,6 +116,11 @@ export const dynamicPluginsFeatureDiscoveryServiceFactory: () => ServiceFactory< 'root' >; +// @public (undocumented) +export interface DynamicPluginsSchemaDiscoveryOptions { + schemaLocator?: (platform: PackagePlatform) => string; +} + // @public (undocumented) export const dynamicPluginsServiceFactory: ( options?: DynamicPluginsFactoryOptions | undefined, @@ -220,5 +226,10 @@ export interface ScannedPluginPackage { manifest: ScannedPluginManifest; } +// @public (undocumented) +export const schemaDiscoveryServiceFactory: ( + options?: DynamicPluginsSchemaDiscoveryOptions | undefined, +) => ServiceFactory; + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-dynamic-feature-service/package.json b/packages/backend-dynamic-feature-service/package.json index 5a3b88eda6..bc99c2a692 100644 --- a/packages/backend-dynamic-feature-service/package.json +++ b/packages/backend-dynamic-feature-service/package.json @@ -32,6 +32,7 @@ "@backstage/cli-common": "workspace:^", "@backstage/cli-node": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", @@ -46,6 +47,7 @@ "@types/express": "^4.17.6", "chokidar": "^3.5.3", "express": "^4.17.1", + "fs-extra": "^7.0.1", "lodash": "^4.17.21", "winston": "^3.2.1" }, @@ -53,7 +55,6 @@ "@backstage/backend-app-api": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", - "@backstage/config-loader": "workspace:^", "wait-for-expect": "^3.0.2" }, "files": [ diff --git a/packages/backend-dynamic-feature-service/src/scanner/index.ts b/packages/backend-dynamic-feature-service/src/scanner/index.ts index c5574cc38e..22b476c143 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/index.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/index.ts @@ -15,3 +15,5 @@ */ export type { ScannedPluginManifest, ScannedPluginPackage } from './types'; +export type { DynamicPluginsSchemaDiscoveryOptions } from './plugin-scanner'; +export { schemaDiscoveryServiceFactory } from './plugin-scanner'; diff --git a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts index af1144f019..7f8996503c 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts @@ -22,7 +22,15 @@ import * as path from 'path'; import * as url from 'url'; import debounce from 'lodash/debounce'; import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + LoggerService, + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import { findPaths } from '@backstage/cli-common'; +import { gatherDynamicPluginsSchemas } from './schemas'; export interface DynamicPluginScannerOptions { config: Config; @@ -315,3 +323,70 @@ export class PluginScanner { this.untrackChanges(); } } + +/** + * @public + */ +export interface DynamicPluginsSchemaDiscoveryOptions { + /** + * Function that returns the plugin-relative path to the Json schema file for a given platform. + * Default behavior is to look for the `configSchema.json` file in the package `dist` sub-directory. + * + * @param platform - The platform of the plugin. + * @returns the plugin-relative path to the Json schema file. + */ + schemaLocator?: (platform: PackagePlatform) => string; +} + +/** + * @public + */ +export const schemaDiscoveryServiceFactory = createServiceFactory( + (options?: DynamicPluginsSchemaDiscoveryOptions) => ({ + service: schemaDiscoveryServiceRef, + deps: { + config: coreServices.rootConfig, + }, + factory({ config }) { + let schemas: ConfigSchemaPackageEntry[] | undefined; + + return { + async getAdditionalSchemas(): Promise<{ + schemas: Array; + }> { + if (schemas) { + return { + schemas, + }; + } + + const logger = { + ...console, + child() { + return this; + }, + }; + + const scanner = PluginScanner.create({ + config, + logger, + // eslint-disable-next-line no-restricted-syntax + backstageRoot: findPaths(__dirname).targetRoot, + preferAlpha: true, + }); + + const { packages } = await scanner.scanRoot(); + + schemas = await gatherDynamicPluginsSchemas( + packages, + logger, + options?.schemaLocator, + ); + return { + schemas, + }; + }, + }; + }, + }), +); diff --git a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts new file mode 100644 index 0000000000..b6d55181a9 --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScannedPluginPackage } from '@backstage/backend-dynamic-feature-service'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import fs from 'fs-extra'; +import * as path from 'path'; +import * as url from 'url'; +import { isEmpty } from 'lodash'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; + +export async function gatherDynamicPluginsSchemas( + packages: ScannedPluginPackage[], + logger: LoggerService, + schemaLocator: (platform: PackagePlatform) => string = () => + path.join('dist', 'configSchema.json'), +): Promise { + const allSchemas: { value: any; path: string }[] = []; + + for (const pluginPackage of packages) { + const platform = PackageRoles.getRoleInfo( + pluginPackage.manifest.backstage.role, + ).platform; + + let pluginLocation = url.fileURLToPath(pluginPackage.location); + if (path.basename(pluginLocation) === 'alpha') { + pluginLocation = path.dirname(pluginLocation); + } + const schemaLocation: string = path.resolve( + pluginLocation, + schemaLocator(platform), + ); + + if (!(await fs.pathExists(schemaLocation))) { + continue; + } + + const serialized = await fs.readJson(schemaLocation); + if (!serialized) { + continue; + } + + if (isEmpty(serialized)) { + continue; + } + + if (!serialized?.$schema || serialized?.type !== 'object') { + logger.error( + `Serialized configuration schema is invalid for plugin ${pluginPackage.manifest.name}`, + ); + continue; + } + + allSchemas.push({ + path: schemaLocation, + value: serialized, + }); + } + + return allSchemas; +} diff --git a/packages/backend-plugin-api/api-report-alpha.md b/packages/backend-plugin-api/api-report-alpha.md index 81b378a671..33b5e748e7 100644 --- a/packages/backend-plugin-api/api-report-alpha.md +++ b/packages/backend-plugin-api/api-report-alpha.md @@ -4,6 +4,7 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -20,5 +21,19 @@ export const featureDiscoveryServiceRef: ServiceRef< 'root' >; +// @alpha (undocumented) +export interface SchemaDiscoveryService { + // (undocumented) + getAdditionalSchemas(): Promise<{ + schemas: Array; + }>; +} + +// @alpha +export const schemaDiscoveryServiceRef: ServiceRef< + SchemaDiscoveryService, + 'root' +>; + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-plugin-api/package.json b/packages/backend-plugin-api/package.json index a2946bd803..5476a75cf3 100644 --- a/packages/backend-plugin-api/package.json +++ b/packages/backend-plugin-api/package.json @@ -47,6 +47,7 @@ "dependencies": { "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/types": "workspace:^", diff --git a/packages/backend-plugin-api/src/alpha.ts b/packages/backend-plugin-api/src/alpha.ts index baee739f49..e5eac99e4a 100644 --- a/packages/backend-plugin-api/src/alpha.ts +++ b/packages/backend-plugin-api/src/alpha.ts @@ -16,8 +16,11 @@ import { BackendFeature, + coreServices, + createServiceFactory, createServiceRef, } from '@backstage/backend-plugin-api'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; /** @alpha */ export interface FeatureDiscoveryService { @@ -33,3 +36,32 @@ export const featureDiscoveryServiceRef = id: 'core.featureDiscovery', scope: 'root', }); + +/** @alpha */ +export interface SchemaDiscoveryService { + getAdditionalSchemas(): Promise<{ schemas: Array }>; +} + +/** + * An optional service that can be used to dynamically load in additional BackendFeatures at runtime. + * @alpha + */ +export const schemaDiscoveryServiceRef = + createServiceRef({ + id: 'core.schemaDiscovery', + scope: 'root', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + config: coreServices.rootConfig, + }, + factory() { + return { + async getAdditionalSchemas() { + return { schemas: [] }; + }, + }; + }, + }), + }); diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 66572dd960..04f1b6a77b 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -45,6 +45,12 @@ export type ConfigSchema = { serialize(): JsonObject; }; +// @public +export type ConfigSchemaPackageEntry = { + value: JsonObject; + path: string; +}; + // @public export type ConfigSchemaProcessingOptions = { visibility?: ConfigVisibility[]; @@ -193,6 +199,7 @@ export type LoadConfigSchemaOptions = ( } ) & { noUndeclaredProperties?: boolean; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; // @public diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index e20e105c7f..80435d6000 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -27,6 +27,7 @@ export type { ConfigVisibility, LoadConfigSchemaOptions, TransformFunc, + ConfigSchemaPackageEntry, } from './schema'; export { loadConfig } from './loader'; export type { diff --git a/packages/config-loader/src/schema/index.ts b/packages/config-loader/src/schema/index.ts index 1dcb9d7b4b..01c18ee905 100644 --- a/packages/config-loader/src/schema/index.ts +++ b/packages/config-loader/src/schema/index.ts @@ -22,4 +22,5 @@ export type { ConfigVisibility, ConfigSchemaProcessingOptions, TransformFunc, + ConfigSchemaPackageEntry, } from './types'; diff --git a/packages/config-loader/src/schema/load.test.ts b/packages/config-loader/src/schema/load.test.ts index 525565f3d5..107f8d1f3b 100644 --- a/packages/config-loader/src/schema/load.test.ts +++ b/packages/config-loader/src/schema/load.test.ts @@ -126,6 +126,60 @@ describe('loadConfigSchema', () => { ); }); + it('should include additional schemas', async () => { + const schema = await loadConfigSchema({ + serialized: { + backstageConfigSchemaVersion: 1, + schemas: [ + { + path: 'mainSchema', + value: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }, + additionalSchemas: [ + { + path: 'additionalSchema', + value: { + type: 'object', + properties: { + additionalKey: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }); + + expect(schema.serialize()).toEqual({ + backstageConfigSchemaVersion: 1, + schemas: [ + { + path: 'mainSchema', + value: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + { + path: 'additionalSchema', + value: { + type: 'object', + properties: { + additionalKey: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + ], + }); + }); + describe('should consider schema', () => { it('when filtering simple config', async () => { mockDir.setContent({ diff --git a/packages/config-loader/src/schema/load.ts b/packages/config-loader/src/schema/load.ts index 885da44116..5c535d1ff6 100644 --- a/packages/config-loader/src/schema/load.ts +++ b/packages/config-loader/src/schema/load.ts @@ -43,6 +43,7 @@ export type LoadConfigSchemaOptions = } ) & { noUndeclaredProperties?: boolean; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; function errorsToError(errors: ValidationError[]): Error { @@ -83,6 +84,7 @@ export async function loadConfigSchema( } schemas = serialized.schemas as ConfigSchemaPackageEntry[]; } + schemas.push(...(options.additionalSchemas || [])); const validate = compileConfigSchemas(schemas, { noUndeclaredProperties: options.noUndeclaredProperties, diff --git a/packages/config-loader/src/schema/types.ts b/packages/config-loader/src/schema/types.ts index 8f679c93fc..66e77018ee 100644 --- a/packages/config-loader/src/schema/types.ts +++ b/packages/config-loader/src/schema/types.ts @@ -18,7 +18,9 @@ import { AppConfig } from '@backstage/config'; import { JsonObject } from '@backstage/types'; /** - * An sub-set of configuration schema. + * A sub-set of configuration schema for a given package. + * + * @public */ export type ConfigSchemaPackageEntry = { /** diff --git a/plugins/app-backend/api-report.md b/plugins/app-backend/api-report.md index 0383129823..9ea42d2fbd 100644 --- a/plugins/app-backend/api-report.md +++ b/plugins/app-backend/api-report.md @@ -4,6 +4,7 @@ ```ts import { Config } from '@backstage/config'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import express from 'express'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -13,6 +14,7 @@ export function createRouter(options: RouterOptions): Promise; // @public (undocumented) export interface RouterOptions { + additionalSchemas?: ConfigSchemaPackageEntry[]; appPackageName: string; // (undocumented) config: Config; diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 63912df5c4..0e223c3650 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -20,6 +20,7 @@ import { Logger } from 'winston'; import { AppConfig, Config } from '@backstage/config'; import { JsonObject } from '@backstage/types'; import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; type InjectOptions = { appConfigs: AppConfig[]; @@ -74,6 +75,7 @@ type ReadOptions = { env: { [name: string]: string | undefined }; appDistDir: string; config: Config; + additionalSchemas?: ConfigSchemaPackageEntry[]; }; /** @@ -90,7 +92,10 @@ export async function readConfigs(options: ReadOptions): Promise { const serializedSchema = await fs.readJson(schemaPath); try { - const schema = await loadConfigSchema({ serialized: serializedSchema }); + const schema = await loadConfigSchema({ + serialized: serializedSchema, + additionalSchemas: options.additionalSchemas, + }); const frontendConfigs = await schema.process( [{ data: config.get() as JsonObject, context: 'app' }], diff --git a/plugins/app-backend/src/service/appPlugin.ts b/plugins/app-backend/src/service/appPlugin.ts index 9ba2ea6d6c..07616e832f 100644 --- a/plugins/app-backend/src/service/appPlugin.ts +++ b/plugins/app-backend/src/service/appPlugin.ts @@ -19,6 +19,7 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; +import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { createRouter } from './router'; import { loggerToWinstonLogger } from '@backstage/backend-common'; import { staticFallbackHandlerExtensionPoint } from '@backstage/plugin-app-node'; @@ -49,8 +50,9 @@ export const appPlugin = createBackendPlugin({ config: coreServices.rootConfig, database: coreServices.database, httpRouter: coreServices.httpRouter, + schemaDiscovery: schemaDiscoveryServiceRef, }, - async init({ logger, config, database, httpRouter }) { + async init({ logger, config, database, httpRouter, schemaDiscovery }) { const appPackageName = config.getOptionalString('app.packageName') ?? 'app'; const winstonLogger = loggerToWinstonLogger(logger); @@ -61,6 +63,9 @@ export const appPlugin = createBackendPlugin({ database, appPackageName, staticFallbackHandler, + additionalSchemas: ( + await schemaDiscovery?.getAdditionalSchemas() + )?.schemas, }); httpRouter.use(router); }, diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index 5574892d37..8e4750dcef 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -37,6 +37,7 @@ import { CACHE_CONTROL_NO_CACHE, CACHE_CONTROL_REVALIDATE_CACHE, } from '../lib/headers'; +import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; // express uses mime v1 while we only have types for mime v2 type Mime = { lookup(arg0: string): string }; @@ -85,6 +86,17 @@ export interface RouterOptions { * This also disables configuration injection though `APP_CONFIG_` environment variables. */ disableConfigInjection?: boolean; + + /** + * + * Provides a list of additional config schemas, in addition to the serialized schemas + * generated during the application build. + * This is useful when additional plugins are dynamically loaded in the application at start, + * which were not part of the application build. This option allows feeding the corresponding + * JSON schemas. + * + */ + additionalSchemas?: ConfigSchemaPackageEntry[]; } /** @public */ @@ -121,6 +133,7 @@ export async function createRouter( config, appDistDir, env: process.env, + additionalSchemas: options.additionalSchemas, }); injectedConfigPath = await injectConfig({ appConfigs, logger, staticDir }); diff --git a/yarn.lock b/yarn.lock index 5f84cbbc51..0e32adda8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3395,6 +3395,7 @@ __metadata: "@types/express": ^4.17.6 chokidar: ^3.5.3 express: ^4.17.1 + fs-extra: ^7.0.1 lodash: ^4.17.21 wait-for-expect: ^3.0.2 winston: ^3.2.1 @@ -3428,6 +3429,7 @@ __metadata: "@backstage/backend-tasks": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/config-loader": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/types": "workspace:^" From 62ba5f404436e72591864ee17d95a290cd96608f Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 22 Jan 2024 15:37:36 +0100 Subject: [PATCH 02/34] Fix review comments: more information in `schemaLocator` Signed-off-by: David Festal --- .../api-report.md | 2 +- .../src/scanner/plugin-scanner.ts | 12 ++++++----- .../src/scanner/schemas.ts | 20 ++++++++----------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/backend-dynamic-feature-service/api-report.md b/packages/backend-dynamic-feature-service/api-report.md index 55a710dbe0..5a9e93f50b 100644 --- a/packages/backend-dynamic-feature-service/api-report.md +++ b/packages/backend-dynamic-feature-service/api-report.md @@ -118,7 +118,7 @@ export const dynamicPluginsFeatureDiscoveryServiceFactory: () => ServiceFactory< // @public (undocumented) export interface DynamicPluginsSchemaDiscoveryOptions { - schemaLocator?: (platform: PackagePlatform) => string; + schemaLocator?: (pluginPackage: ScannedPluginPackage) => string; } // @public (undocumented) diff --git a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts index 7f8996503c..0c6ad90e52 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts @@ -329,13 +329,15 @@ export class PluginScanner { */ export interface DynamicPluginsSchemaDiscoveryOptions { /** - * Function that returns the plugin-relative path to the Json schema file for a given platform. - * Default behavior is to look for the `configSchema.json` file in the package `dist` sub-directory. + * Function that returns the path to the Json schema file for a given scanned plugin package. + * The path is either absolute, or relative to the plugin package root directory. * - * @param platform - The platform of the plugin. - * @returns the plugin-relative path to the Json schema file. + * Default behavior is to look for the `dist/configSchema.json` relative path. + * + * @param pluginPackage - The scanned plugin package. + * @returns the absolute or plugin-relative path to the Json schema file. */ - schemaLocator?: (platform: PackagePlatform) => string; + schemaLocator?: (pluginPackage: ScannedPluginPackage) => string; } /** diff --git a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts index b6d55181a9..ebb456e63b 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts @@ -21,29 +21,25 @@ import * as path from 'path'; import * as url from 'url'; import { isEmpty } from 'lodash'; import { LoggerService } from '@backstage/backend-plugin-api'; -import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; export async function gatherDynamicPluginsSchemas( packages: ScannedPluginPackage[], logger: LoggerService, - schemaLocator: (platform: PackagePlatform) => string = () => + schemaLocator: (pluginPackage: ScannedPluginPackage) => string = () => path.join('dist', 'configSchema.json'), ): Promise { const allSchemas: { value: any; path: string }[] = []; for (const pluginPackage of packages) { - const platform = PackageRoles.getRoleInfo( - pluginPackage.manifest.backstage.role, - ).platform; + let schemaLocation = schemaLocator(pluginPackage); - let pluginLocation = url.fileURLToPath(pluginPackage.location); - if (path.basename(pluginLocation) === 'alpha') { - pluginLocation = path.dirname(pluginLocation); + if (!path.isAbsolute(schemaLocation)) { + let pluginLocation = url.fileURLToPath(pluginPackage.location); + if (path.basename(pluginLocation) === 'alpha') { + pluginLocation = path.dirname(pluginLocation); + } + schemaLocation = path.resolve(pluginLocation, schemaLocation); } - const schemaLocation: string = path.resolve( - pluginLocation, - schemaLocator(platform), - ); if (!(await fs.pathExists(schemaLocation))) { continue; From 54ad8e1553a1b89dbb5ce97ffd94edf91bd77206 Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 22 Jan 2024 18:43:41 +0100 Subject: [PATCH 03/34] Fix review comment: split changesets... and use `patch` instead of `minor` since there are no breaking changes. Signed-off-by: David Festal --- .changeset/late-poets-love.md | 6 ++++++ .changeset/nine-forks-sell.md | 6 ++++++ .changeset/rich-bees-fry.md | 5 +++++ .changeset/swift-pumpkins-shake.md | 9 +-------- 4 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 .changeset/late-poets-love.md create mode 100644 .changeset/nine-forks-sell.md create mode 100644 .changeset/rich-bees-fry.md diff --git a/.changeset/late-poets-love.md b/.changeset/late-poets-love.md new file mode 100644 index 0000000000..664c665a0d --- /dev/null +++ b/.changeset/late-poets-love.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-app-api': patch +'@backstage/backend-common': patch +--- + +Make the root logger use the additional configuration schemas when hiding secrets. diff --git a/.changeset/nine-forks-sell.md b/.changeset/nine-forks-sell.md new file mode 100644 index 0000000000..9886be0022 --- /dev/null +++ b/.changeset/nine-forks-sell.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-plugin-api': patch +'@backstage/config-loader': patch +--- + +Introduce a way to provide additional configuration schemas at runtime (application start). diff --git a/.changeset/rich-bees-fry.md b/.changeset/rich-bees-fry.md new file mode 100644 index 0000000000..841ae985fc --- /dev/null +++ b/.changeset/rich-bees-fry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-app-backend': minor +--- + +Make the `app-backend` plugin use additional configuration schemas. diff --git a/.changeset/swift-pumpkins-shake.md b/.changeset/swift-pumpkins-shake.md index c9441a3690..231039d1d8 100644 --- a/.changeset/swift-pumpkins-shake.md +++ b/.changeset/swift-pumpkins-shake.md @@ -1,12 +1,5 @@ --- -'@backstage/backend-app-api': minor -'@backstage/backend-common': minor -'@backstage/config-loader': minor -'@backstage/plugin-app-backend': minor '@backstage/backend-dynamic-feature-service': minor --- -Allow referencing additional configuration schemas at runtime (application start), and have them taken in account where schemas are used: - -- the backed-app plugin, in order to prepare the frontend configuration -- the backend application loggers, in order to hide secret configuration values (defined in the schemas). +Implement the discovery of additional configuration schemas for dynamic plugins. From 84fcbc49d6d444bcfb347886ed8ca33cb139954b Mon Sep 17 00:00:00 2001 From: David Festal Date: Tue, 6 Feb 2024 16:43:27 +0100 Subject: [PATCH 04/34] Fix review comment: don't use `ConfigSchemaPackageEntry` nor make it public. Signed-off-by: David Festal --- packages/backend-app-api/api-report.md | 5 ++-- packages/backend-app-api/src/config/config.ts | 4 +-- .../rootLoggerServiceFactory.test.ts | 28 ++++++++----------- packages/backend-common/api-report.md | 6 ++-- packages/backend-common/src/config.test.ts | 19 ++++++------- packages/backend-common/src/config.ts | 8 ++---- .../src/scanner/plugin-scanner.ts | 6 ++-- .../src/scanner/schemas.ts | 11 +++----- .../backend-plugin-api/api-report-alpha.md | 6 ++-- packages/backend-plugin-api/src/alpha.ts | 8 ++++-- packages/config-loader/api-report.md | 10 ++----- packages/config-loader/src/index.ts | 1 - .../config-loader/src/schema/load.test.ts | 15 ++++------ packages/config-loader/src/schema/load.ts | 13 +++++++-- packages/config-loader/src/schema/types.ts | 4 +-- plugins/app-backend/api-report.md | 6 ++-- plugins/app-backend/src/lib/config.ts | 3 +- plugins/app-backend/src/service/router.ts | 6 ++-- 18 files changed, 76 insertions(+), 83 deletions(-) diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index 1143beec15..cff8c3c0cd 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -9,7 +9,6 @@ import type { AppConfig } from '@backstage/config'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CacheClient } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { CorsOptions } from 'cors'; import { DiscoveryService } from '@backstage/backend-plugin-api'; import { ErrorRequestHandler } from 'express'; @@ -65,7 +64,9 @@ export const cacheServiceFactory: () => ServiceFactory; export function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { + [context: string]: JsonObject; + }; }): Promise<(config: Config) => Iterable>; // @public diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index d4eb5fccb9..abca385e62 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -23,19 +23,19 @@ import { loadConfig, ConfigTarget, LoadConfigOptionsRemote, - ConfigSchemaPackageEntry, } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import type { Config, AppConfig } from '@backstage/config'; import { getPackages } from '@manypkg/get-packages'; import { ObservableConfigProxy } from './ObservableConfigProxy'; import { isValidUrl } from '../lib/urls'; +import { JsonObject } from '@backstage/types'; /** @public */ export async function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { [context: string]: JsonObject }; }): Promise<(config: Config) => Iterable> { const { logger, dir = process.cwd() } = options; const { packages } = await getPackages(dir); diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts index ac7b9e8cf2..b840822c6f 100644 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts @@ -22,15 +22,12 @@ import { createServiceFactory, } from '@backstage/backend-plugin-api'; import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; -import { - ConfigSchemaPackageEntry, - ConfigSources, - StaticConfigSource, -} from '@backstage/config-loader'; +import { ConfigSources, StaticConfigSource } from '@backstage/config-loader'; import { transports } from 'winston'; import { rootLifecycleServiceFactory } from '../rootLifecycle'; import { lifecycleServiceFactory } from '../lifecycle'; import { loggerServiceFactory } from '../logger'; +import { JsonObject } from '@backstage/types'; describe('rootLogger', () => { describe('rootLoggerServiceFactory', () => { @@ -46,20 +43,17 @@ describe('rootLogger', () => { }, }; - const additionalSchemas = [ - { - path: 'test', - value: { - type: 'object', - properties: { - secretValue: { - type: 'string', - visibility: 'secret', - }, + const additionalSchemas = { + test: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', }, }, }, - ]; + }; const logs: string[] = []; jest @@ -97,7 +91,7 @@ describe('rootLogger', () => { }, factory: async () => ({ getAdditionalSchemas: async (): Promise<{ - schemas: Array; + schemas: { [context: string]: JsonObject }; }> => ({ schemas: additionalSchemas, }), diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 7a3fc72a5b..607eaa010e 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -20,7 +20,6 @@ import { CacheService as CacheClient } from '@backstage/backend-plugin-api'; import { CacheServiceOptions as CacheClientOptions } from '@backstage/backend-plugin-api'; import { CacheServiceSetOptions as CacheClientSetOptions } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import cors from 'cors'; import Docker from 'dockerode'; import { ErrorRequestHandler } from 'express'; @@ -33,6 +32,7 @@ import { GitLabIntegration } from '@backstage/integration'; import { HostDiscovery as HostDiscovery_2 } from '@backstage/backend-app-api'; import { IdentityService } from '@backstage/backend-plugin-api'; import { isChildPath } from '@backstage/cli-common'; +import { JsonObject } from '@backstage/types'; import { Knex } from 'knex'; import knexFactory from 'knex'; import { KubeConfig } from '@kubernetes/client-node'; @@ -561,7 +561,9 @@ export function loadBackendConfig(options: { logger: LoggerService; remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { + [context: string]: JsonObject; + }; argv: string[]; watch?: boolean; }): Promise; diff --git a/packages/backend-common/src/config.test.ts b/packages/backend-common/src/config.test.ts index 931c0b5c84..05d1cbc3ec 100644 --- a/packages/backend-common/src/config.test.ts +++ b/packages/backend-common/src/config.test.ts @@ -36,20 +36,17 @@ describe('config', () => { }, ]; - const additionalSchemas = [ - { - path: 'test', - value: { - type: 'object', - properties: { - secretValue: { - type: 'string', - visibility: 'secret', - }, + const additionalSchemas = { + test: { + type: 'object', + properties: { + secretValue: { + type: 'string', + visibility: 'secret', }, }, }, - ]; + }; const logs: string[] = []; jest diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index b5d1d2b896..ac89f3f4ec 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -20,11 +20,9 @@ import { } from '@backstage/backend-app-api'; import { LoggerService } from '@backstage/backend-plugin-api'; import { AppConfig, Config } from '@backstage/config'; -import { - ConfigSchemaPackageEntry, - LoadConfigOptionsRemote, -} from '@backstage/config-loader'; +import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; +import { JsonObject } from '@backstage/types'; /** * Load configuration for a Backend. @@ -38,7 +36,7 @@ export async function loadBackendConfig(options: { // process.argv or any other overrides remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { [context: string]: JsonObject }; argv: string[]; watch?: boolean; }): Promise { diff --git a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts index 0c6ad90e52..a9e11ca60b 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts @@ -28,9 +28,9 @@ import { createServiceFactory, } from '@backstage/backend-plugin-api'; import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import { findPaths } from '@backstage/cli-common'; import { gatherDynamicPluginsSchemas } from './schemas'; +import { JsonObject } from '@backstage/types'; export interface DynamicPluginScannerOptions { config: Config; @@ -350,11 +350,11 @@ export const schemaDiscoveryServiceFactory = createServiceFactory( config: coreServices.rootConfig, }, factory({ config }) { - let schemas: ConfigSchemaPackageEntry[] | undefined; + let schemas: { [context: string]: JsonObject } | undefined; return { async getAdditionalSchemas(): Promise<{ - schemas: Array; + schemas: { [context: string]: JsonObject }; }> { if (schemas) { return { diff --git a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts index ebb456e63b..f00828859a 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts @@ -15,20 +15,20 @@ */ import { ScannedPluginPackage } from '@backstage/backend-dynamic-feature-service'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import fs from 'fs-extra'; import * as path from 'path'; import * as url from 'url'; import { isEmpty } from 'lodash'; import { LoggerService } from '@backstage/backend-plugin-api'; +import { JsonObject } from '@backstage/types'; export async function gatherDynamicPluginsSchemas( packages: ScannedPluginPackage[], logger: LoggerService, schemaLocator: (pluginPackage: ScannedPluginPackage) => string = () => path.join('dist', 'configSchema.json'), -): Promise { - const allSchemas: { value: any; path: string }[] = []; +): Promise<{ [context: string]: JsonObject }> { + const allSchemas: { [context: string]: JsonObject } = {}; for (const pluginPackage of packages) { let schemaLocation = schemaLocator(pluginPackage); @@ -61,10 +61,7 @@ export async function gatherDynamicPluginsSchemas( continue; } - allSchemas.push({ - path: schemaLocation, - value: serialized, - }); + allSchemas[schemaLocation] = serialized; } return allSchemas; diff --git a/packages/backend-plugin-api/api-report-alpha.md b/packages/backend-plugin-api/api-report-alpha.md index 33b5e748e7..0d9eaf4a0e 100644 --- a/packages/backend-plugin-api/api-report-alpha.md +++ b/packages/backend-plugin-api/api-report-alpha.md @@ -4,7 +4,7 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import { JsonObject } from '@backstage/types'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -25,7 +25,9 @@ export const featureDiscoveryServiceRef: ServiceRef< export interface SchemaDiscoveryService { // (undocumented) getAdditionalSchemas(): Promise<{ - schemas: Array; + schemas: { + [context: string]: JsonObject; + }; }>; } diff --git a/packages/backend-plugin-api/src/alpha.ts b/packages/backend-plugin-api/src/alpha.ts index e5eac99e4a..7d7e3177c2 100644 --- a/packages/backend-plugin-api/src/alpha.ts +++ b/packages/backend-plugin-api/src/alpha.ts @@ -20,7 +20,7 @@ import { createServiceFactory, createServiceRef, } from '@backstage/backend-plugin-api'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import { JsonObject } from '@backstage/types'; /** @alpha */ export interface FeatureDiscoveryService { @@ -39,7 +39,9 @@ export const featureDiscoveryServiceRef = /** @alpha */ export interface SchemaDiscoveryService { - getAdditionalSchemas(): Promise<{ schemas: Array }>; + getAdditionalSchemas(): Promise<{ + schemas: { [context: string]: JsonObject }; + }>; } /** @@ -59,7 +61,7 @@ export const schemaDiscoveryServiceRef = factory() { return { async getAdditionalSchemas() { - return { schemas: [] }; + return { schemas: {} }; }, }; }, diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 04f1b6a77b..c3fc9f522e 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -45,12 +45,6 @@ export type ConfigSchema = { serialize(): JsonObject; }; -// @public -export type ConfigSchemaPackageEntry = { - value: JsonObject; - path: string; -}; - // @public export type ConfigSchemaProcessingOptions = { visibility?: ConfigVisibility[]; @@ -199,7 +193,9 @@ export type LoadConfigSchemaOptions = ( } ) & { noUndeclaredProperties?: boolean; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { + [context: string]: JsonObject; + }; }; // @public diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 80435d6000..e20e105c7f 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -27,7 +27,6 @@ export type { ConfigVisibility, LoadConfigSchemaOptions, TransformFunc, - ConfigSchemaPackageEntry, } from './schema'; export { loadConfig } from './loader'; export type { diff --git a/packages/config-loader/src/schema/load.test.ts b/packages/config-loader/src/schema/load.test.ts index 107f8d1f3b..f9299340c3 100644 --- a/packages/config-loader/src/schema/load.test.ts +++ b/packages/config-loader/src/schema/load.test.ts @@ -142,17 +142,14 @@ describe('loadConfigSchema', () => { }, ], }, - additionalSchemas: [ - { - path: 'additionalSchema', - value: { - type: 'object', - properties: { - additionalKey: { type: 'string', visibility: 'frontend' }, - }, + additionalSchemas: { + additionalSchema: { + type: 'object', + properties: { + additionalKey: { type: 'string', visibility: 'frontend' }, }, }, - ], + }, }); expect(schema.serialize()).toEqual({ diff --git a/packages/config-loader/src/schema/load.ts b/packages/config-loader/src/schema/load.ts index 5c535d1ff6..03b85b2756 100644 --- a/packages/config-loader/src/schema/load.ts +++ b/packages/config-loader/src/schema/load.ts @@ -43,7 +43,7 @@ export type LoadConfigSchemaOptions = } ) & { noUndeclaredProperties?: boolean; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { [context: string]: JsonObject }; }; function errorsToError(errors: ValidationError[]): Error { @@ -84,7 +84,16 @@ export async function loadConfigSchema( } schemas = serialized.schemas as ConfigSchemaPackageEntry[]; } - schemas.push(...(options.additionalSchemas || [])); + if (options.additionalSchemas) { + schemas.push( + ...Object.keys(options.additionalSchemas).map(context => { + return { + path: context, + value: options.additionalSchemas![context], + }; + }), + ); + } const validate = compileConfigSchemas(schemas, { noUndeclaredProperties: options.noUndeclaredProperties, diff --git a/packages/config-loader/src/schema/types.ts b/packages/config-loader/src/schema/types.ts index 66e77018ee..8f679c93fc 100644 --- a/packages/config-loader/src/schema/types.ts +++ b/packages/config-loader/src/schema/types.ts @@ -18,9 +18,7 @@ import { AppConfig } from '@backstage/config'; import { JsonObject } from '@backstage/types'; /** - * A sub-set of configuration schema for a given package. - * - * @public + * An sub-set of configuration schema. */ export type ConfigSchemaPackageEntry = { /** diff --git a/plugins/app-backend/api-report.md b/plugins/app-backend/api-report.md index 9ea42d2fbd..fd85b68a87 100644 --- a/plugins/app-backend/api-report.md +++ b/plugins/app-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { Config } from '@backstage/config'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; import express from 'express'; +import { JsonObject } from '@backstage/types'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -14,7 +14,9 @@ export function createRouter(options: RouterOptions): Promise; // @public (undocumented) export interface RouterOptions { - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { + [context: string]: JsonObject; + }; appPackageName: string; // (undocumented) config: Config; diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 0e223c3650..dc041a2593 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -20,7 +20,6 @@ import { Logger } from 'winston'; import { AppConfig, Config } from '@backstage/config'; import { JsonObject } from '@backstage/types'; import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; type InjectOptions = { appConfigs: AppConfig[]; @@ -75,7 +74,7 @@ type ReadOptions = { env: { [name: string]: string | undefined }; appDistDir: string; config: Config; - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { [context: string]: JsonObject }; }; /** diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index 8e4750dcef..204d5df26a 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -37,7 +37,7 @@ import { CACHE_CONTROL_NO_CACHE, CACHE_CONTROL_REVALIDATE_CACHE, } from '../lib/headers'; -import { ConfigSchemaPackageEntry } from '@backstage/config-loader'; +import { JsonObject } from '@backstage/types'; // express uses mime v1 while we only have types for mime v2 type Mime = { lookup(arg0: string): string }; @@ -89,14 +89,14 @@ export interface RouterOptions { /** * - * Provides a list of additional config schemas, in addition to the serialized schemas + * Provides a map of additional config schemas, in addition to the serialized schemas * generated during the application build. * This is useful when additional plugins are dynamically loaded in the application at start, * which were not part of the application build. This option allows feeding the corresponding * JSON schemas. * */ - additionalSchemas?: ConfigSchemaPackageEntry[]; + additionalSchemas?: { [context: string]: JsonObject }; } /** @public */ From dd5d7cc92479bb7c541fa04b8abbf61a3472ce5d Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Wed, 7 Feb 2024 13:54:18 +0100 Subject: [PATCH 05/34] Wrap adaptV4Theme with createV5Theme to support extra vars Signed-off-by: Ilya Savich --- .changeset/hungry-days-remain.md | 5 +++++ packages/theme/src/unified/UnifiedTheme.tsx | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/hungry-days-remain.md diff --git a/.changeset/hungry-days-remain.md b/.changeset/hungry-days-remain.md new file mode 100644 index 0000000000..b0c3642a35 --- /dev/null +++ b/.changeset/hungry-days-remain.md @@ -0,0 +1,5 @@ +--- +'@backstage/theme': patch +--- + +Fixed missing extra variables like `applyDarkStyles` in Mui V5 theme after calling `createUnifiedThemeFromV4` function diff --git a/packages/theme/src/unified/UnifiedTheme.tsx b/packages/theme/src/unified/UnifiedTheme.tsx index aeae9694ee..d6612d0a1d 100644 --- a/packages/theme/src/unified/UnifiedTheme.tsx +++ b/packages/theme/src/unified/UnifiedTheme.tsx @@ -93,5 +93,5 @@ export function createUnifiedThemeFromV4( ): UnifiedTheme { const v5Theme = adaptV4Theme(options as DeprecatedThemeOptions); const v4Theme = createTheme(options); - return new UnifiedThemeHolder(v4Theme, v5Theme); + return new UnifiedThemeHolder(v4Theme, createV5Theme(v5Theme)); } From c3bc4bc8e6d5bc39b9ed20ae809840af43d36149 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Tue, 6 Feb 2024 10:57:26 -0500 Subject: [PATCH 06/34] Fixed documentation of Dynatrace plugin configuration options Signed-off-by: Florian JUDITH --- plugins/dynatrace/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/dynatrace/README.md b/plugins/dynatrace/README.md index 5e2394cee3..73ac545265 100644 --- a/plugins/dynatrace/README.md +++ b/plugins/dynatrace/README.md @@ -62,10 +62,11 @@ This plugin requires a proxy endpoint for Dynatrace configured in `app-config.ya ```yaml proxy: - '/dynatrace': - target: 'https://example.dynatrace.com/api/v2' - headers: - Authorization: 'Api-Token ${DYNATRACE_ACCESS_TOKEN}' + endpoints: + '/dynatrace': + target: 'https://example.dynatrace.com/api/v2' + headers: + Authorization: 'Api-Token ${DYNATRACE_ACCESS_TOKEN}' ``` It also requires a `baseUrl` for rendering links to problems in the table like so: From 0930c9ed68f5e08339c8d1299304979e7ca2f886 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Wed, 7 Feb 2024 19:40:08 -0500 Subject: [PATCH 07/34] Fixed typo Signed-off-by: Florian JUDITH --- .changeset/spicy-bobcats-return.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/spicy-bobcats-return.md diff --git a/.changeset/spicy-bobcats-return.md b/.changeset/spicy-bobcats-return.md new file mode 100644 index 0000000000..9bd4b33784 --- /dev/null +++ b/.changeset/spicy-bobcats-return.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-dynatrace': patch +--- + +Fixed Dynatrace plugin proxy configuration From 76888725223543c87076d9d7204192c78ff8c90d Mon Sep 17 00:00:00 2001 From: rpigu-i Date: Thu, 8 Feb 2024 15:54:44 -0500 Subject: [PATCH 08/34] Update to Flightcontrol deployment docs Signed-off-by: rpigu-i --- docs/deployment/flightcontrol.md | 157 ++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 2 deletions(-) diff --git a/docs/deployment/flightcontrol.md b/docs/deployment/flightcontrol.md index 482e9341ea..82e7199ee3 100644 --- a/docs/deployment/flightcontrol.md +++ b/docs/deployment/flightcontrol.md @@ -9,8 +9,159 @@ This guide explains how to deploy Backstage to [Flightcontrol](https://www.fligh Before you begin, make sure you have a [Flightcontrol account](https://app.flightcontrol.dev/signup?ref=backstage) and a [Github account](https://github.com/login) to follow this guide. +# Creating a Dockerfile and .dockerignore + +Once your Flightcontrol account is setup, you will need to prepare a `Dockerfile` and `.dockerignore` to deploy Backstage to your target AWS environment. The standard `Dockerfile` included in the Backstage repository assumes that the `skeleton.tar.gz` and `bundle.tar.gz` already exist. When integrating Flightcontrol directly with a GitHub repository, changes to files within this repository will automatically start a Flightcontrol deployment, unless you have configured deployments via a webbook. Regardless of which method you have chosen, when Flightcontrol starts a deployment process it will pull all files directly from the repository, and not from a GitHub runner or other location. As Flightcontrol does not contain CI steps this results in the zip files having to be commited to the repository in order to be available for the `Dockerfile` to copy. + +A simple work around for this is to use a `Dockerfile` with a [multi-stage build](https://docs.docker.com/build/building/multi-stage/). Using this approach we can create our zip files in the first build stage, and copy them into the second stage ready for deployment. + +The following two code snippets demonstrate this method. The first block of code is an example of a multi-stage build approach and should be added to a file called `Dockerfile.flightcontrol` in the `packages/backend` directory. + +``` +# This dockerfile builds an image for the backend package. +# It should be executed with the root of the repo as docker context. +# +# Before building this image, be sure to have run the following commands in the repo root: +# +# yarn install +# yarn tsc +# yarn build:backend +# +# Once the commands have been run, you can build the image using `yarn build-image` +FROM public.ecr.aws/c4g3p9t9/node-bookworm-slim:18.16.1-8.6.0 as build + +USER node +WORKDIR /app +COPY --chown=node:node . . + +RUN yarn install --frozen-lockfile + +# tsc outputs type definitions to dist-types/ in the repo root, which are then consumed by the build +RUN yarn tsc + +# Build the backend, which bundles it all up into the packages/backend/dist folder. +# The configuration files here should match the one you use inside the Dockerfile below. +RUN yarn build:backend --config ../../app-config.yaml + +FROM public.ecr.aws/c4g3p9t9/node-bookworm-slim:18.16.1-8.6.0 + +# Install isolate-vm dependencies, these are needed by the @backstage/plugin-scaffolder-backend. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends python3 g++ build-essential && \ + yarn config set python /usr/bin/python3 + +# Install sqlite3 dependencies. You can skip this if you don't use sqlite3 in the image, +# in which case you should also move better-sqlite3 to "devDependencies" in package.json. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends libsqlite3-dev + +# From here on we use the least-privileged `node` user to run the backend. +USER node + +# This should create the app dir as `node`. +# If it is instead created as `root` then the `tar` command below will fail: `can't create directory 'packages/': Permission denied`. +# If this occurs, then ensure BuildKit is enabled (`DOCKER_BUILDKIT=1`) so the app dir is correctly created as `node`. +WORKDIR /app + +# This switches many Node.js dependencies to production mode. +ENV NODE_ENV production + +# Copy repo skeleton first, to avoid unnecessary docker cache invalidation. +# The skeleton contains the package.json of each package in the monorepo, +# and along with yarn.lock and the root package.json, that's enough to run yarn install. +COPY --from=build app/yarn.lock app/package.json app/packages/backend/dist/skeleton.tar.gz ./ +RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz + +RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ + yarn install --frozen-lockfile --production --network-timeout 300000 + +# Then copy the rest of the backend bundle, along with any other files we might want. +COPY --from=build app/packages/backend/dist/bundle.tar.gz app/app-config*.yaml ./ +RUN tar xzf bundle.tar.gz && rm bundle.tar.gz + +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] +``` + +In order to prevent the default `.dockerignore` file from preventing the copying of the relevant files into the first stage of our build, we need to override it with a custom one. This can also be added to the `packages/backend` directory, it should be called `Dockerfile.flightcontrol.dockerignore` and contain the following. + +``` +.git +.yarn/cache +.yarn/install-state.gz +node_modules +packages/*/node_modules +*.local.yaml +``` + +With these two new files, you will now have the necessary steps in place to build the zip files, copy them into the second stage of the build, and deploy them via Flightcontrol. + +# Creating a custom app-config.production file + +Our next steps before switching to the Flightcontrol dashboard is to create a custom app-config.production file. The purposes of this is to update how Backstage will connect to the RDS hosted Postgres DB. The default `app-config.production` file uses the `host`, `port`, `user` and `password` environment variables. + +When creating a database via the Flightcontrol dashboard, we are given an environment variable with the single connection string. Therefore we need an `app-config.production` file which uses this variable. + +Within your Backstage project create a new file called `app-config.production.flightcontrol.yaml`. Add the following configuration to it: + +``` +app: + # Should be the same as backend.baseUrl when using the `app-backend` plugin. + baseUrl: http://localhost:7007 + +backend: + # Note that the baseUrl should be the URL that the browser and other clients + # should use when communicating with the backend, i.e. it needs to be + # reachable not just from within the backend host, but from all of your + # callers. When its value is "http://localhost:7007", it's strictly private + # and can't be reached by others. + baseUrl: http://localhost:7007 + # The listener can also be expressed as a single : string. In this case we bind to + # all interfaces, the most permissive setting. The right value depends on your specific deployment. + listen: ':7007' + + # config options: https://node-postgres.com/api/client + database: + client: pg + pluginDivisionMode: 'schema' + connection: + connectionString: ${POSTGRES_CON_STRING} + ssl: + require: true + rejectUnauthorized: false + + # Flightcontrol provides a single DB connection string + # environment variable which can be renamed to POSTGRES_CON_STRING. + # the following default values are therefore not used: + # host: ${POSTGRES_HOST} + # port: ${POSTGRES_PORT} + # user: ${POSTGRES_USER} + # password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl + # you can set the sslmode configuration option via the `PGSSLMODE` environment variable + # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) + # ssl: + # ca: # if you have a CA file and want to verify it you can uncomment this section + # $file: /ca/server.crt + +catalog: + # Overrides the default list locations from app-config.yaml as these contain example data. + # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details + # on how to get entities into the catalog. + locations: [] +``` + +Make a note of the environment variable used in the `connectionString`. In our example above this is `POSTGRES_CON_STRING`. You will need this later when you have deployed the database via the Flightcontrol console, and wish to connect to it from Backstage. + # Deployment Via Dashboard +Ensure that custom `Dockerfile.flightcontrol`, `Dockerfile.flightcontrol.dockerignore` and `app-config.production.flightcontrol.yaml` have been commited to your repository before following the next steps. + +Login into the Flightcontrol Dashboard and complete each of the following: + 1. Create a new project from the Flightcontrol Dashboard 2. Select the GitHub repo for your Backstage project @@ -25,7 +176,7 @@ Before you begin, make sure you have a [Flightcontrol account](https://app.fligh | Health Check Path | /catalog | | Port | 7007 | -5. Click `Create Project` and complete any required steps (like linking your AWS account). +5. Click `Create Project` and complete any required steps (like linking your AWS account). Remember to point the project to your custom files. # Deployment via Code @@ -33,7 +184,7 @@ Before you begin, make sure you have a [Flightcontrol account](https://app.fligh 2. Select the GitHub repo for your Backstage project -3. Select the `flightcontrol.json` Config Type. +3. Select the `flightcontrol.json` Config Type. Update the example JSON below where applicable to your custom Dockerfile. ```json filename="flightcontrol.json" { @@ -72,6 +223,8 @@ Before you begin, make sure you have a [Flightcontrol account](https://app.fligh If you need a database or Redis for your Backstage plugins, you can easily add those to your Flightcontrol deployment. For more information, see [the flightcontrol docs](https://www.flightcontrol.dev/docs/guides/flightcontrol/using-code?ref=backstage#redis). +When creating a Postgres RDS database you will need to update the connection string variable name to the one included in the `app-config.production.flightcontrol.yaml`. As noted above in our example we called this `POSTGRES_CON_STRING`. + ## Troubleshooting - [Flightcontrol Documentation](https://www.flightcontrol.dev/docs?ref=backstage) From 4717467a871df6669cce0639ca96734287381bce Mon Sep 17 00:00:00 2001 From: rpigu-i Date: Thu, 8 Feb 2024 16:04:32 -0500 Subject: [PATCH 09/34] Update to Flightcontrol deployment docs syntax highlighting Signed-off-by: rpigu-i --- docs/deployment/flightcontrol.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/deployment/flightcontrol.md b/docs/deployment/flightcontrol.md index 82e7199ee3..b7ca9f3578 100644 --- a/docs/deployment/flightcontrol.md +++ b/docs/deployment/flightcontrol.md @@ -17,7 +17,7 @@ A simple work around for this is to use a `Dockerfile` with a [multi-stage build The following two code snippets demonstrate this method. The first block of code is an example of a multi-stage build approach and should be added to a file called `Dockerfile.flightcontrol` in the `packages/backend` directory. -``` +```dockerfile filename="Dockerfile.flightcontrol" # This dockerfile builds an image for the backend package. # It should be executed with the root of the repo as docker context. # @@ -88,7 +88,7 @@ CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app In order to prevent the default `.dockerignore` file from preventing the copying of the relevant files into the first stage of our build, we need to override it with a custom one. This can also be added to the `packages/backend` directory, it should be called `Dockerfile.flightcontrol.dockerignore` and contain the following. -``` +```ssh filename="Dockerfile.flightcontrol.dockerignore" .git .yarn/cache .yarn/install-state.gz @@ -107,7 +107,7 @@ When creating a database via the Flightcontrol dashboard, we are given an enviro Within your Backstage project create a new file called `app-config.production.flightcontrol.yaml`. Add the following configuration to it: -``` +```yaml filename="app-config.production.flightcontrol.yaml" app: # Should be the same as backend.baseUrl when using the `app-backend` plugin. baseUrl: http://localhost:7007 From 2505a39209658011fce13526764e9f0dea7da88e Mon Sep 17 00:00:00 2001 From: rpigu-i Date: Thu, 8 Feb 2024 20:55:03 -0500 Subject: [PATCH 10/34] Fix minor typos in Flightcontrol documentation Signed-off-by: rpigu-i --- docs/deployment/flightcontrol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deployment/flightcontrol.md b/docs/deployment/flightcontrol.md index b7ca9f3578..1923e5bc39 100644 --- a/docs/deployment/flightcontrol.md +++ b/docs/deployment/flightcontrol.md @@ -11,7 +11,7 @@ Before you begin, make sure you have a [Flightcontrol account](https://app.fligh # Creating a Dockerfile and .dockerignore -Once your Flightcontrol account is setup, you will need to prepare a `Dockerfile` and `.dockerignore` to deploy Backstage to your target AWS environment. The standard `Dockerfile` included in the Backstage repository assumes that the `skeleton.tar.gz` and `bundle.tar.gz` already exist. When integrating Flightcontrol directly with a GitHub repository, changes to files within this repository will automatically start a Flightcontrol deployment, unless you have configured deployments via a webbook. Regardless of which method you have chosen, when Flightcontrol starts a deployment process it will pull all files directly from the repository, and not from a GitHub runner or other location. As Flightcontrol does not contain CI steps this results in the zip files having to be commited to the repository in order to be available for the `Dockerfile` to copy. +Once your Flightcontrol account is setup, you will need to prepare a `Dockerfile` and `.dockerignore` to deploy Backstage to your target AWS environment. The standard `Dockerfile` included in the Backstage repository assumes that the `skeleton.tar.gz` and `bundle.tar.gz` already exist. When integrating Flightcontrol directly with a GitHub repository, changes to files within this repository will automatically start a Flightcontrol deployment, unless you have configured deployments via a webhook. Regardless of which method you have chosen, when Flightcontrol starts a deployment process it will pull all files directly from the repository, and not from a GitHub runner or other location. As Flightcontrol does not contain CI steps this results in the zip files having to be committed to the repository in order to be available for the `Dockerfile` to copy. A simple work around for this is to use a `Dockerfile` with a [multi-stage build](https://docs.docker.com/build/building/multi-stage/). Using this approach we can create our zip files in the first build stage, and copy them into the second stage ready for deployment. @@ -158,7 +158,7 @@ Make a note of the environment variable used in the `connectionString`. In our e # Deployment Via Dashboard -Ensure that custom `Dockerfile.flightcontrol`, `Dockerfile.flightcontrol.dockerignore` and `app-config.production.flightcontrol.yaml` have been commited to your repository before following the next steps. +Ensure that custom `Dockerfile.flightcontrol`, `Dockerfile.flightcontrol.dockerignore` and `app-config.production.flightcontrol.yaml` have been committed to your repository before following the next steps. Login into the Flightcontrol Dashboard and complete each of the following: From 0aeab6f770d6fd0c173408d67f68bcfea5f5ea35 Mon Sep 17 00:00:00 2001 From: rpigu-i Date: Fri, 9 Feb 2024 07:24:58 -0500 Subject: [PATCH 11/34] Update image in Dockerfile example in Flighcontrol docs to node:18-bookworm-slim Signed-off-by: rpigu-i --- docs/deployment/flightcontrol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deployment/flightcontrol.md b/docs/deployment/flightcontrol.md index 1923e5bc39..6078d931fb 100644 --- a/docs/deployment/flightcontrol.md +++ b/docs/deployment/flightcontrol.md @@ -28,7 +28,7 @@ The following two code snippets demonstrate this method. The first block of code # yarn build:backend # # Once the commands have been run, you can build the image using `yarn build-image` -FROM public.ecr.aws/c4g3p9t9/node-bookworm-slim:18.16.1-8.6.0 as build +FROM node:18-bookworm-slim as build USER node WORKDIR /app @@ -43,7 +43,7 @@ RUN yarn tsc # The configuration files here should match the one you use inside the Dockerfile below. RUN yarn build:backend --config ../../app-config.yaml -FROM public.ecr.aws/c4g3p9t9/node-bookworm-slim:18.16.1-8.6.0 +FROM node:18-bookworm-slim # Install isolate-vm dependencies, these are needed by the @backstage/plugin-scaffolder-backend. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ From bc28b37689b5ca7856d3ba324b6ac754c4112960 Mon Sep 17 00:00:00 2001 From: Jean-Louis FEREY Date: Fri, 9 Feb 2024 14:24:22 +0100 Subject: [PATCH 12/34] Update README.md for package dependencies configuration Signed-off-by: Jean-Louis FEREY --- plugins/devtools/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/devtools/README.md b/plugins/devtools/README.md index f6db8d1417..2d21e8e6f2 100644 --- a/plugins/devtools/README.md +++ b/plugins/devtools/README.md @@ -418,8 +418,8 @@ By default, only packages with names starting with `@backstage` and `@internal` devTools: info: packagePrefixes: - - @roadiehq/backstage- - - @spotify/backstage- + - '@roadiehq/backstage-' + - '@spotify/backstage-' ``` ### External Dependencies Configuration From 1c250188c79c0ce8023b8ba7882b90d7f37196ed Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Fri, 26 Jan 2024 14:51:04 -0500 Subject: [PATCH 13/34] fix: Updates the imports in frontend docs Signed-off-by: Darshan Simha --- .../software-templates/writing-custom-step-layouts.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-custom-step-layouts.md b/docs/features/software-templates/writing-custom-step-layouts.md index b965f60f6b..d3c7c8fe82 100644 --- a/docs/features/software-templates/writing-custom-step-layouts.md +++ b/docs/features/software-templates/writing-custom-step-layouts.md @@ -20,11 +20,11 @@ The [createScaffolderLayout](https://backstage.io/docs/reference/plugin-scaffold ```ts import React from 'react'; +import { scaffolderPlugin } from '@backstage/plugin-scaffolder'; import { createScaffolderLayout, LayoutTemplate, - scaffolderPlugin, -} from '@backstage/plugin-scaffolder'; +} from '@backstage/plugin-scaffolder-react'; import { Grid } from '@material-ui/core'; const TwoColumn: LayoutTemplate = ({ properties, description, title }) => { From 41c4fc9d415af4c95e3a88f6a4dcd3a1a7cd342b Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Fri, 2 Feb 2024 09:08:10 -0500 Subject: [PATCH 14/34] fix: critical fix to documentation for fixing the issue with k8's deployments Signed-off-by: Darshan Simha --- docs/deployment/k8s.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index b0218c8657..dee972f84d 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -397,6 +397,19 @@ backend: password: ${POSTGRES_PASSWORD} ``` +Please also change the host and port environment variables in the `app-config.production.yaml` to the following: + +```yaml +backend: + database: + client: pg + connection: + host: ${POSTGRES_SERVICE_HOST} + port: ${POSTGRES_SERVICE_PORT} + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} +``` + Make sure to rebuild the Docker image after applying `app-config.yaml` changes. Apply this Deployment to the Kubernetes cluster: From 1e52ba229ef4df694a0f77df5b8882f3a7873452 Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Fri, 2 Feb 2024 09:17:47 -0500 Subject: [PATCH 15/34] fix: removed changes from another branch Signed-off-by: Darshan Simha --- .../software-templates/writing-custom-step-layouts.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-custom-step-layouts.md b/docs/features/software-templates/writing-custom-step-layouts.md index d3c7c8fe82..b965f60f6b 100644 --- a/docs/features/software-templates/writing-custom-step-layouts.md +++ b/docs/features/software-templates/writing-custom-step-layouts.md @@ -20,11 +20,11 @@ The [createScaffolderLayout](https://backstage.io/docs/reference/plugin-scaffold ```ts import React from 'react'; -import { scaffolderPlugin } from '@backstage/plugin-scaffolder'; import { createScaffolderLayout, LayoutTemplate, -} from '@backstage/plugin-scaffolder-react'; + scaffolderPlugin, +} from '@backstage/plugin-scaffolder'; import { Grid } from '@material-ui/core'; const TwoColumn: LayoutTemplate = ({ properties, description, title }) => { From 662fd66dd8878ba2c1f5abe8609ff7ee7a6c08f4 Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Sun, 4 Feb 2024 20:21:46 -0500 Subject: [PATCH 16/34] fix: updated the message based on comments Signed-off-by: Darshan Simha --- docs/deployment/k8s.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index dee972f84d..e7fd025026 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -384,7 +384,7 @@ $ yarn build-image --tag backstage:1.0.0 There is no special wiring needed to access the PostgreSQL service. Since it's running on the same cluster, Kubernetes will inject `POSTGRES_SERVICE_HOST` and `POSTGRES_SERVICE_PORT` environment variables into our Backstage container. -These can be used in the Backstage `app-config.yaml` along with the secrets: +These can be used in the Backstage `app-config.yaml` along with the secrets. Apply this to app-config.production.yaml as well if you have one: ```yaml backend: From 45996b83451b6134d73a19ee64a0444ef67bfbdc Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Thu, 8 Feb 2024 13:02:30 -0500 Subject: [PATCH 17/34] fix: updated based on review comment Signed-off-by: Darshan Simha --- docs/deployment/k8s.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index e7fd025026..4596608caf 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -399,17 +399,6 @@ backend: Please also change the host and port environment variables in the `app-config.production.yaml` to the following: -```yaml -backend: - database: - client: pg - connection: - host: ${POSTGRES_SERVICE_HOST} - port: ${POSTGRES_SERVICE_PORT} - user: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} -``` - Make sure to rebuild the Docker image after applying `app-config.yaml` changes. Apply this Deployment to the Kubernetes cluster: From 12b1d5eacc7ee55379e58d4b2435491dbc84900a Mon Sep 17 00:00:00 2001 From: Darshan Simha Date: Thu, 8 Feb 2024 14:14:54 -0500 Subject: [PATCH 18/34] fix: pr comments Signed-off-by: Darshan Simha --- docs/deployment/k8s.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index 4596608caf..3dd2b8227e 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -397,8 +397,6 @@ backend: password: ${POSTGRES_PASSWORD} ``` -Please also change the host and port environment variables in the `app-config.production.yaml` to the following: - Make sure to rebuild the Docker image after applying `app-config.yaml` changes. Apply this Deployment to the Kubernetes cluster: From a4b7bc4a6baac52d6fcc8ef9a80575b3c3e6972c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Feb 2024 15:25:18 +0100 Subject: [PATCH 19/34] beps/0003: update discussion issue link Signed-off-by: Patrik Oldsberg --- beps/0003-auth-architecture-evolution/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beps/0003-auth-architecture-evolution/README.md b/beps/0003-auth-architecture-evolution/README.md index ec2660a9cc..2dbb6e0666 100644 --- a/beps/0003-auth-architecture-evolution/README.md +++ b/beps/0003-auth-architecture-evolution/README.md @@ -14,7 +14,7 @@ creation-date: 2024-01-28 -[**Discussion Issue**](https://github.com/backstage/backstage/issues/NNNNN) +[**Discussion Issue**](https://github.com/backstage/backstage/issues/22605) - [Summary](#summary) - [Motivation](#motivation) From bd29b2870ba71717adc24d52b6cf36c788cd37cb Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 10 Feb 2024 12:33:55 -0600 Subject: [PATCH 20/34] Added `experimentalExtraAllowedOrigins` to config Signed-off-by: Andre Wanlin --- plugins/auth-backend/config.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/auth-backend/config.d.ts b/plugins/auth-backend/config.d.ts index 25cd8d88ec..075c5e7942 100644 --- a/plugins/auth-backend/config.d.ts +++ b/plugins/auth-backend/config.d.ts @@ -191,5 +191,9 @@ export interface Config { */ backstageTokenExpiration?: HumanDuration; }; + /** + * Additional app origins to allow for authenticating + */ + experimentalExtraAllowedOrigins?: string; }; } From 8321c9750f19ea8029f0bff7617dd6036fb5f6b9 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 10 Feb 2024 12:34:51 -0600 Subject: [PATCH 21/34] Added changeset Signed-off-by: Andre Wanlin --- .changeset/sour-ligers-hunt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sour-ligers-hunt.md diff --git a/.changeset/sour-ligers-hunt.md b/.changeset/sour-ligers-hunt.md new file mode 100644 index 0000000000..5642494c24 --- /dev/null +++ b/.changeset/sour-ligers-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Added `experimentalExtraAllowedOrigins` to config From 785ff247f60ccbdc0bdcf32e9776f59e6613e3c6 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 10 Feb 2024 13:23:05 -0600 Subject: [PATCH 22/34] Added `validateLocationsExist` to the config Signed-off-by: Andre Wanlin --- .changeset/mean-apes-wait.md | 5 +++++ plugins/catalog-backend-module-github/config.d.ts | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/mean-apes-wait.md diff --git a/.changeset/mean-apes-wait.md b/.changeset/mean-apes-wait.md new file mode 100644 index 0000000000..838f1f2286 --- /dev/null +++ b/.changeset/mean-apes-wait.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added `validateLocationsExist` to the config diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index ad4d2d6ec5..8b0f266c14 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -135,6 +135,11 @@ export interface Config { * Default: `/catalog-info.yaml`. */ catalogPath?: string; + /** + * (Optional) Whether to validate locations that exist before emitting them. + * Default: `false`. + */ + validateLocationsExist?: boolean; /** * (Optional) Filter configuration. */ From 425488bac5c23643069c4647871ec307d1e7661e Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 10 Feb 2024 14:42:54 -0600 Subject: [PATCH 23/34] Updated to be string[] Signed-off-by: Andre Wanlin --- plugins/auth-backend/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/config.d.ts b/plugins/auth-backend/config.d.ts index 075c5e7942..4c8430b33f 100644 --- a/plugins/auth-backend/config.d.ts +++ b/plugins/auth-backend/config.d.ts @@ -194,6 +194,6 @@ export interface Config { /** * Additional app origins to allow for authenticating */ - experimentalExtraAllowedOrigins?: string; + experimentalExtraAllowedOrigins?: string[]; }; } From 1ab22c40e01d3a58d5fa24b2d75716218d79e129 Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Fri, 2 Feb 2024 08:31:39 +0200 Subject: [PATCH 24/34] feat: allow defining signal type this makes using especially the frontend API much usable as the signal can be typed instead having plain json object to play around with. Signed-off-by: Heikki Hellgren --- .changeset/six-emus-buy.md | 7 +++++ plugins/signals-node/api-report.md | 12 ++++++--- .../signals-node/src/DefaultSignalService.ts | 11 ++++---- plugins/signals-node/src/SignalService.ts | 8 ++++-- plugins/signals-node/src/types.ts | 4 +-- plugins/signals-react/api-report.md | 10 ++++--- plugins/signals-react/src/api/SignalApi.ts | 4 +-- plugins/signals-react/src/hooks/useSignal.ts | 15 +++++++---- plugins/signals/api-report.md | 9 +++---- plugins/signals/src/api/SignalClient.ts | 26 ++++++++++--------- 10 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 .changeset/six-emus-buy.md diff --git a/.changeset/six-emus-buy.md b/.changeset/six-emus-buy.md new file mode 100644 index 0000000000..1ef2a8e124 --- /dev/null +++ b/.changeset/six-emus-buy.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-signals-react': patch +'@backstage/plugin-signals-node': patch +'@backstage/plugin-signals': patch +--- + +Allow defining signal type to publish and receive diff --git a/plugins/signals-node/api-report.md b/plugins/signals-node/api-report.md index cbf4142daf..4c95891171 100644 --- a/plugins/signals-node/api-report.md +++ b/plugins/signals-node/api-report.md @@ -11,19 +11,23 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; export class DefaultSignalService implements SignalService { // (undocumented) static create(options: SignalServiceOptions): DefaultSignalService; - publish(signal: SignalPayload): Promise; + publish( + signal: SignalPayload, + ): Promise; } // @public (undocumented) -export type SignalPayload = { +export type SignalPayload = { recipients: string[] | string | null; channel: string; - message: JsonObject; + message: SignalType; }; // @public (undocumented) export interface SignalService { - publish(signal: SignalPayload): Promise; + publish( + signal: SignalPayload, + ): Promise; } // @public (undocumented) diff --git a/plugins/signals-node/src/DefaultSignalService.ts b/plugins/signals-node/src/DefaultSignalService.ts index 2a234fe5d4..d99bae48ff 100644 --- a/plugins/signals-node/src/DefaultSignalService.ts +++ b/plugins/signals-node/src/DefaultSignalService.ts @@ -16,6 +16,7 @@ import { EventBroker } from '@backstage/plugin-events-node'; import { SignalPayload, SignalServiceOptions } from './types'; import { SignalService } from './SignalService'; +import { JsonObject } from '@backstage/types'; /** @public */ export class DefaultSignalService implements SignalService { @@ -31,12 +32,12 @@ export class DefaultSignalService implements SignalService { } /** - * Publishes a message to user refs to specific topic - * @param recipients - string or array of user ref strings to publish message to - * @param topic - message topic - * @param message - message to publish + * Publishes a signal to user refs to specific topic + * @param signal - Signal to publish */ - async publish(signal: SignalPayload) { + async publish( + signal: SignalPayload, + ) { await this.eventBroker?.publish({ topic: 'signals', eventPayload: signal, diff --git a/plugins/signals-node/src/SignalService.ts b/plugins/signals-node/src/SignalService.ts index 7d021ccf4e..383da51b4c 100644 --- a/plugins/signals-node/src/SignalService.ts +++ b/plugins/signals-node/src/SignalService.ts @@ -14,11 +14,15 @@ * limitations under the License. */ import { SignalPayload } from './types'; +import { JsonObject } from '@backstage/types'; /** @public */ export interface SignalService { /** - * Publishes a message to user refs to specific topic + * Publishes a signal to user refs to specific topic + * @param signal - Signal to publish */ - publish(signal: SignalPayload): Promise; + publish( + signal: SignalPayload, + ): Promise; } diff --git a/plugins/signals-node/src/types.ts b/plugins/signals-node/src/types.ts index 7e61aea320..bebee835ed 100644 --- a/plugins/signals-node/src/types.ts +++ b/plugins/signals-node/src/types.ts @@ -24,8 +24,8 @@ export type SignalServiceOptions = { }; /** @public */ -export type SignalPayload = { +export type SignalPayload = { recipients: string[] | string | null; channel: string; - message: JsonObject; + message: SignalType; }; diff --git a/plugins/signals-react/api-report.md b/plugins/signals-react/api-report.md index ef4bc1b7fc..0c555cd566 100644 --- a/plugins/signals-react/api-report.md +++ b/plugins/signals-react/api-report.md @@ -9,9 +9,9 @@ import { JsonObject } from '@backstage/types'; // @public (undocumented) export interface SignalApi { // (undocumented) - subscribe( + subscribe( channel: string, - onMessage: (message: JsonObject) => void, + onMessage: (message: SignalType) => void, ): SignalSubscriber; } @@ -25,8 +25,10 @@ export interface SignalSubscriber { } // @public (undocumented) -export const useSignal: (channel: string) => { - lastSignal: JsonObject | null; +export const useSignal: ( + channel: string, +) => { + lastSignal: SignalType | null; isSignalsAvailable: boolean; }; diff --git a/plugins/signals-react/src/api/SignalApi.ts b/plugins/signals-react/src/api/SignalApi.ts index b67b2ea0dc..f27c8d46ee 100644 --- a/plugins/signals-react/src/api/SignalApi.ts +++ b/plugins/signals-react/src/api/SignalApi.ts @@ -28,8 +28,8 @@ export interface SignalSubscriber { /** @public */ export interface SignalApi { - subscribe( + subscribe( channel: string, - onMessage: (message: JsonObject) => void, + onMessage: (message: SignalType) => void, ): SignalSubscriber; } diff --git a/plugins/signals-react/src/hooks/useSignal.ts b/plugins/signals-react/src/hooks/useSignal.ts index 084427e2cc..db65454e0b 100644 --- a/plugins/signals-react/src/hooks/useSignal.ts +++ b/plugins/signals-react/src/hooks/useSignal.ts @@ -19,18 +19,23 @@ import { JsonObject } from '@backstage/types'; import { useEffect, useMemo, useState } from 'react'; /** @public */ -export const useSignal = (channel: string) => { +export const useSignal = ( + channel: string, +): { lastSignal: SignalType | null; isSignalsAvailable: boolean } => { const apiHolder = useApiHolder(); // Use apiHolder instead useApi in case signalApi is not available in the // backstage instance this is used const signals = apiHolder.get(signalApiRef); - const [lastSignal, setLastSignal] = useState(null); + const [lastSignal, setLastSignal] = useState(null); useEffect(() => { let unsub: null | (() => void) = null; if (signals) { - const { unsubscribe } = signals.subscribe(channel, (msg: JsonObject) => { - setLastSignal(msg); - }); + const { unsubscribe } = signals.subscribe( + channel, + (msg: SignalType) => { + setLastSignal(msg); + }, + ); unsub = unsubscribe; } return () => { diff --git a/plugins/signals/api-report.md b/plugins/signals/api-report.md index 398d5fbfe6..180b3076ec 100644 --- a/plugins/signals/api-report.md +++ b/plugins/signals/api-report.md @@ -8,6 +8,7 @@ import { DiscoveryApi } from '@backstage/core-plugin-api'; import { IdentityApi } from '@backstage/core-plugin-api'; import { JsonObject } from '@backstage/types'; import { SignalApi } from '@backstage/plugin-signals-react'; +import { SignalSubscriber } from '@backstage/plugin-signals-react'; // @public (undocumented) export class SignalClient implements SignalApi { @@ -23,12 +24,10 @@ export class SignalClient implements SignalApi { // (undocumented) static readonly DEFAULT_RECONNECT_TIMEOUT_MS: number; // (undocumented) - subscribe( + subscribe( channel: string, - onMessage: (message: JsonObject) => void, - ): { - unsubscribe: () => void; - }; + onMessage: (message: SignalType) => void, + ): SignalSubscriber; } // @public (undocumented) diff --git a/plugins/signals/src/api/SignalClient.ts b/plugins/signals/src/api/SignalClient.ts index d6c4634264..806646c351 100644 --- a/plugins/signals/src/api/SignalClient.ts +++ b/plugins/signals/src/api/SignalClient.ts @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SignalApi } from '@backstage/plugin-signals-react'; +import { SignalApi, SignalSubscriber } from '@backstage/plugin-signals-react'; import { JsonObject } from '@backstage/types'; import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { v4 as uuid } from 'uuid'; type Subscription = { channel: string; - callback: (message: JsonObject) => void; + callback: (message: any) => void; }; const WS_CLOSE_NORMAL = 1000; @@ -62,16 +62,16 @@ export class SignalClient implements SignalApi { private reconnectTimeout: number, ) {} - subscribe( + subscribe( channel: string, - onMessage: (message: JsonObject) => void, - ): { unsubscribe: () => void } { + onMessage: (message: SignalType) => void, + ): SignalSubscriber { const subscriptionId = uuid(); const exists = [...this.subscriptions.values()].find( sub => sub.channel === channel, ); this.subscriptions.set(subscriptionId, { - channel: channel, + channel, callback: onMessage, }); @@ -178,12 +178,14 @@ export class SignalClient implements SignalApi { private handleMessage(data: MessageEvent) { try { - const json = JSON.parse(data.data) as JsonObject; - if (json.channel) { - for (const sub of this.subscriptions.values()) { - if (sub.channel === json.channel) { - sub.callback(json.message as JsonObject); - } + const json = JSON.parse(data.data); + if (!json.channel) { + return; + } + + for (const sub of this.subscriptions.values()) { + if (sub.channel === json.channel) { + sub.callback(json.message); } } } catch (e) { From 1bbb0471d00c8c6dbb25175e1d80d21a78019e00 Mon Sep 17 00:00:00 2001 From: drodil Date: Mon, 12 Feb 2024 14:48:10 +0200 Subject: [PATCH 25/34] Update plugins/signals-node/api-report.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: drodil --- plugins/signals-node/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/signals-node/api-report.md b/plugins/signals-node/api-report.md index 4c95891171..6636ae2c31 100644 --- a/plugins/signals-node/api-report.md +++ b/plugins/signals-node/api-report.md @@ -17,7 +17,7 @@ export class DefaultSignalService implements SignalService { } // @public (undocumented) -export type SignalPayload = { +export type SignalPayload = { recipients: string[] | string | null; channel: string; message: SignalType; From ca8081166bd9b54691c9aa33a5a762b041f5fc5e Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Mon, 12 Feb 2024 15:28:41 +0200 Subject: [PATCH 26/34] chore: signal type naming Signed-off-by: Heikki Hellgren --- plugins/signals-node/api-report.md | 10 +++++----- plugins/signals-node/src/DefaultSignalService.ts | 4 ++-- plugins/signals-node/src/SignalService.ts | 4 ++-- plugins/signals-node/src/types.ts | 4 ++-- plugins/signals-react/api-report.md | 8 ++++---- plugins/signals-react/src/api/SignalApi.ts | 4 ++-- plugins/signals-react/src/hooks/useSignal.ts | 10 +++++----- plugins/signals/api-report.md | 4 ++-- plugins/signals/src/api/SignalClient.ts | 4 ++-- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/plugins/signals-node/api-report.md b/plugins/signals-node/api-report.md index 6636ae2c31..9d67e24732 100644 --- a/plugins/signals-node/api-report.md +++ b/plugins/signals-node/api-report.md @@ -11,8 +11,8 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; export class DefaultSignalService implements SignalService { // (undocumented) static create(options: SignalServiceOptions): DefaultSignalService; - publish( - signal: SignalPayload, + publish( + signal: SignalPayload, ): Promise; } @@ -20,13 +20,13 @@ export class DefaultSignalService implements SignalService { export type SignalPayload = { recipients: string[] | string | null; channel: string; - message: SignalType; + message: TMessage; }; // @public (undocumented) export interface SignalService { - publish( - signal: SignalPayload, + publish( + signal: SignalPayload, ): Promise; } diff --git a/plugins/signals-node/src/DefaultSignalService.ts b/plugins/signals-node/src/DefaultSignalService.ts index d99bae48ff..d1bc14e166 100644 --- a/plugins/signals-node/src/DefaultSignalService.ts +++ b/plugins/signals-node/src/DefaultSignalService.ts @@ -35,8 +35,8 @@ export class DefaultSignalService implements SignalService { * Publishes a signal to user refs to specific topic * @param signal - Signal to publish */ - async publish( - signal: SignalPayload, + async publish( + signal: SignalPayload, ) { await this.eventBroker?.publish({ topic: 'signals', diff --git a/plugins/signals-node/src/SignalService.ts b/plugins/signals-node/src/SignalService.ts index 383da51b4c..7011c6c525 100644 --- a/plugins/signals-node/src/SignalService.ts +++ b/plugins/signals-node/src/SignalService.ts @@ -22,7 +22,7 @@ export interface SignalService { * Publishes a signal to user refs to specific topic * @param signal - Signal to publish */ - publish( - signal: SignalPayload, + publish( + signal: SignalPayload, ): Promise; } diff --git a/plugins/signals-node/src/types.ts b/plugins/signals-node/src/types.ts index bebee835ed..ec6e815b29 100644 --- a/plugins/signals-node/src/types.ts +++ b/plugins/signals-node/src/types.ts @@ -24,8 +24,8 @@ export type SignalServiceOptions = { }; /** @public */ -export type SignalPayload = { +export type SignalPayload = { recipients: string[] | string | null; channel: string; - message: SignalType; + message: TMessage; }; diff --git a/plugins/signals-react/api-report.md b/plugins/signals-react/api-report.md index 0c555cd566..87d856288b 100644 --- a/plugins/signals-react/api-report.md +++ b/plugins/signals-react/api-report.md @@ -9,9 +9,9 @@ import { JsonObject } from '@backstage/types'; // @public (undocumented) export interface SignalApi { // (undocumented) - subscribe( + subscribe( channel: string, - onMessage: (message: SignalType) => void, + onMessage: (message: TMessage) => void, ): SignalSubscriber; } @@ -25,10 +25,10 @@ export interface SignalSubscriber { } // @public (undocumented) -export const useSignal: ( +export const useSignal: ( channel: string, ) => { - lastSignal: SignalType | null; + lastSignal: TMessage | null; isSignalsAvailable: boolean; }; diff --git a/plugins/signals-react/src/api/SignalApi.ts b/plugins/signals-react/src/api/SignalApi.ts index f27c8d46ee..66f2c7e881 100644 --- a/plugins/signals-react/src/api/SignalApi.ts +++ b/plugins/signals-react/src/api/SignalApi.ts @@ -28,8 +28,8 @@ export interface SignalSubscriber { /** @public */ export interface SignalApi { - subscribe( + subscribe( channel: string, - onMessage: (message: SignalType) => void, + onMessage: (message: TMessage) => void, ): SignalSubscriber; } diff --git a/plugins/signals-react/src/hooks/useSignal.ts b/plugins/signals-react/src/hooks/useSignal.ts index db65454e0b..6fb0f4696f 100644 --- a/plugins/signals-react/src/hooks/useSignal.ts +++ b/plugins/signals-react/src/hooks/useSignal.ts @@ -19,20 +19,20 @@ import { JsonObject } from '@backstage/types'; import { useEffect, useMemo, useState } from 'react'; /** @public */ -export const useSignal = ( +export const useSignal = ( channel: string, -): { lastSignal: SignalType | null; isSignalsAvailable: boolean } => { +): { lastSignal: TMessage | null; isSignalsAvailable: boolean } => { const apiHolder = useApiHolder(); // Use apiHolder instead useApi in case signalApi is not available in the // backstage instance this is used const signals = apiHolder.get(signalApiRef); - const [lastSignal, setLastSignal] = useState(null); + const [lastSignal, setLastSignal] = useState(null); useEffect(() => { let unsub: null | (() => void) = null; if (signals) { - const { unsubscribe } = signals.subscribe( + const { unsubscribe } = signals.subscribe( channel, - (msg: SignalType) => { + (msg: TMessage) => { setLastSignal(msg); }, ); diff --git a/plugins/signals/api-report.md b/plugins/signals/api-report.md index 180b3076ec..1af1063433 100644 --- a/plugins/signals/api-report.md +++ b/plugins/signals/api-report.md @@ -24,9 +24,9 @@ export class SignalClient implements SignalApi { // (undocumented) static readonly DEFAULT_RECONNECT_TIMEOUT_MS: number; // (undocumented) - subscribe( + subscribe( channel: string, - onMessage: (message: SignalType) => void, + onMessage: (message: TMessage) => void, ): SignalSubscriber; } diff --git a/plugins/signals/src/api/SignalClient.ts b/plugins/signals/src/api/SignalClient.ts index 806646c351..feccf4a105 100644 --- a/plugins/signals/src/api/SignalClient.ts +++ b/plugins/signals/src/api/SignalClient.ts @@ -62,9 +62,9 @@ export class SignalClient implements SignalApi { private reconnectTimeout: number, ) {} - subscribe( + subscribe( channel: string, - onMessage: (message: SignalType) => void, + onMessage: (message: TMessage) => void, ): SignalSubscriber { const subscriptionId = uuid(); const exists = [...this.subscriptions.values()].find( From 2fb527dd34dab57917005d6e0b51a1c5332d4f99 Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 12 Feb 2024 19:13:01 +0100 Subject: [PATCH 27/34] Fix review comments: remove `schemaDiscovery` service Signed-off-by: David Festal --- .changeset/late-poets-love.md | 3 +- .changeset/nine-forks-sell.md | 6 - .changeset/rich-bees-fry.md | 3 +- .changeset/swift-pumpkins-shake.md | 5 +- packages/backend-app-api/api-report.md | 5 +- packages/backend-app-api/src/config/config.ts | 13 +- .../rootLoggerServiceFactory.test.ts | 128 ------------ .../rootLogger/rootLoggerServiceFactory.ts | 11 +- packages/backend-common/api-report.md | 4 - packages/backend-common/src/config.test.ts | 84 -------- packages/backend-common/src/config.ts | 3 - .../api-report.md | 32 ++- .../package.json | 19 +- .../src/index.ts | 1 + .../src/scanner/index.ts | 2 - .../src/scanner/plugin-scanner.ts | 79 +------- .../src/scanner/schemas.ts | 68 ------- .../src/schemas/appBackendModule.ts | 53 +++++ .../src/schemas/index.ts | 24 +++ .../src/schemas/rootLoggerServiceFactory.ts | 63 ++++++ .../src/schemas/schemas.ts | 187 ++++++++++++++++++ .../backend-plugin-api/api-report-alpha.md | 17 -- packages/backend-plugin-api/package.json | 1 - packages/backend-plugin-api/src/alpha.ts | 34 ---- packages/config-loader/api-report.md | 3 - packages/config-loader/src/schema/index.ts | 1 - .../config-loader/src/schema/load.test.ts | 51 ----- packages/config-loader/src/schema/load.ts | 11 -- plugins/app-backend/api-report.md | 6 +- plugins/app-backend/src/lib/config.ts | 17 +- plugins/app-backend/src/service/appPlugin.ts | 27 ++- plugins/app-backend/src/service/router.ts | 12 +- plugins/app-node/api-report.md | 14 ++ plugins/app-node/package.json | 4 +- plugins/app-node/src/extensions.ts | 23 +++ plugins/app-node/src/index.ts | 4 + plugins/app-node/src/schema.ts | 39 ++++ yarn.lock | 7 +- 38 files changed, 513 insertions(+), 551 deletions(-) delete mode 100644 .changeset/nine-forks-sell.md delete mode 100644 packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts delete mode 100644 packages/backend-common/src/config.test.ts delete mode 100644 packages/backend-dynamic-feature-service/src/scanner/schemas.ts create mode 100644 packages/backend-dynamic-feature-service/src/schemas/appBackendModule.ts create mode 100644 packages/backend-dynamic-feature-service/src/schemas/index.ts create mode 100644 packages/backend-dynamic-feature-service/src/schemas/rootLoggerServiceFactory.ts create mode 100644 packages/backend-dynamic-feature-service/src/schemas/schemas.ts create mode 100644 plugins/app-node/src/schema.ts diff --git a/.changeset/late-poets-love.md b/.changeset/late-poets-love.md index 664c665a0d..bd6692560d 100644 --- a/.changeset/late-poets-love.md +++ b/.changeset/late-poets-love.md @@ -1,6 +1,5 @@ --- '@backstage/backend-app-api': patch -'@backstage/backend-common': patch --- -Make the root logger use the additional configuration schemas when hiding secrets. +Allow the `createConfigSecretEnumerator` to take an optional `configSchema` argument with an already-loaded global configuration schema. diff --git a/.changeset/nine-forks-sell.md b/.changeset/nine-forks-sell.md deleted file mode 100644 index 9886be0022..0000000000 --- a/.changeset/nine-forks-sell.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/backend-plugin-api': patch -'@backstage/config-loader': patch ---- - -Introduce a way to provide additional configuration schemas at runtime (application start). diff --git a/.changeset/rich-bees-fry.md b/.changeset/rich-bees-fry.md index 841ae985fc..3dcfa95532 100644 --- a/.changeset/rich-bees-fry.md +++ b/.changeset/rich-bees-fry.md @@ -1,5 +1,6 @@ --- '@backstage/plugin-app-backend': minor +'@backstage/plugin-app-node': minor --- -Make the `app-backend` plugin use additional configuration schemas. +Allow the `app-backend` plugin to use a global configuration schema provided externally through an extension. diff --git a/.changeset/swift-pumpkins-shake.md b/.changeset/swift-pumpkins-shake.md index 231039d1d8..c7a0a44ade 100644 --- a/.changeset/swift-pumpkins-shake.md +++ b/.changeset/swift-pumpkins-shake.md @@ -2,4 +2,7 @@ '@backstage/backend-dynamic-feature-service': minor --- -Implement the discovery of additional configuration schemas for dynamic plugins. +Implement the discovery of additional individual configuration schemas for dynamic plugins, and provide: + +- an alternate implementation of the root logger service that takes them into account, +- an extension to the App backend plugin to set a global configuration schema that takes them into account. diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index cff8c3c0cd..f41a38de7b 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -9,6 +9,7 @@ import type { AppConfig } from '@backstage/config'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CacheClient } from '@backstage/backend-common'; import { Config } from '@backstage/config'; +import { ConfigSchema } from '@backstage/config-loader'; import { CorsOptions } from 'cors'; import { DiscoveryService } from '@backstage/backend-plugin-api'; import { ErrorRequestHandler } from 'express'; @@ -64,9 +65,7 @@ export const cacheServiceFactory: () => ServiceFactory; export function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; - additionalSchemas?: { - [context: string]: JsonObject; - }; + schema?: ConfigSchema; }): Promise<(config: Config) => Iterable>; // @public diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index abca385e62..8f98c5597c 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -23,26 +23,27 @@ import { loadConfig, ConfigTarget, LoadConfigOptionsRemote, + ConfigSchema, } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import type { Config, AppConfig } from '@backstage/config'; import { getPackages } from '@manypkg/get-packages'; import { ObservableConfigProxy } from './ObservableConfigProxy'; import { isValidUrl } from '../lib/urls'; -import { JsonObject } from '@backstage/types'; /** @public */ export async function createConfigSecretEnumerator(options: { logger: LoggerService; dir?: string; - additionalSchemas?: { [context: string]: JsonObject }; + schema?: ConfigSchema; }): Promise<(config: Config) => Iterable> { const { logger, dir = process.cwd() } = options; const { packages } = await getPackages(dir); - const schema = await loadConfigSchema({ - dependencies: packages.map(p => p.packageJson.name), - additionalSchemas: options.additionalSchemas, - }); + const schema = + options.schema ?? + (await loadConfigSchema({ + dependencies: packages.map(p => p.packageJson.name).filter(() => false), + })); return (config: Config) => { const [secretsData] = schema.process( diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts deleted file mode 100644 index b840822c6f..0000000000 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { createSpecializedBackend } from '../../../wiring'; -import { rootLoggerServiceFactory } from './rootLoggerServiceFactory'; -import { - coreServices, - createBackendPlugin, - createServiceFactory, -} from '@backstage/backend-plugin-api'; -import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; -import { ConfigSources, StaticConfigSource } from '@backstage/config-loader'; -import { transports } from 'winston'; -import { rootLifecycleServiceFactory } from '../rootLifecycle'; -import { lifecycleServiceFactory } from '../lifecycle'; -import { loggerServiceFactory } from '../logger'; -import { JsonObject } from '@backstage/types'; - -describe('rootLogger', () => { - describe('rootLoggerServiceFactory', () => { - afterEach(() => { - jest.resetModules(); - }); - - it('also hide secret config values coming from additional schemas', async () => { - const additionalConfigs = { - context: 'test', - data: { - secretValue: 'shouldBeHidden', - }, - }; - - const additionalSchemas = { - test: { - type: 'object', - properties: { - secretValue: { - type: 'string', - visibility: 'secret', - }, - }, - }, - }; - - const logs: string[] = []; - jest - .spyOn(transports.Console.prototype, 'log') - .mockImplementation((s: any, next: any) => { - logs.push(s.message); - if (next) { - next(); - } - return undefined; - }); - - const backend = createSpecializedBackend({ - defaultServiceFactories: [ - lifecycleServiceFactory(), - loggerServiceFactory(), - rootLifecycleServiceFactory(), - rootLoggerServiceFactory(), - createServiceFactory({ - service: coreServices.rootConfig, - deps: {}, - async factory() { - return await ConfigSources.toConfig( - ConfigSources.merge([ - ConfigSources.default({}), - StaticConfigSource.create(additionalConfigs), - ]), - ); - }, - }), - createServiceFactory({ - service: schemaDiscoveryServiceRef, - deps: { - config: coreServices.rootConfig, - }, - factory: async () => ({ - getAdditionalSchemas: async (): Promise<{ - schemas: { [context: string]: JsonObject }; - }> => ({ - schemas: additionalSchemas, - }), - }), - }), - ], - }); - backend.add( - createBackendPlugin({ - pluginId: 'test', - register(env) { - env.registerInit({ - deps: { - logger: coreServices.logger, - lifecycle: coreServices.lifecycle, - }, - async init({ logger }) { - logger.info('test'); - logger.info('shouldBeHidden'); - }, - }); - }, - }), - ); - await backend.start(); - - expect(logs[0]).toMatch( - /Found \d+ new secrets in config that will be redacted/, - ); - expect(logs[1]).toEqual('test'); - expect(logs[2]).toEqual('[REDACTED]'); - }, 30000); - }); -}); diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts index becf618f4f..bd45df383f 100644 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerServiceFactory.ts @@ -18,7 +18,6 @@ import { createServiceFactory, coreServices, } from '@backstage/backend-plugin-api'; -import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { WinstonLogger } from '../../../logging'; import { transports, format } from 'winston'; import { createConfigSecretEnumerator } from '../../../config'; @@ -28,9 +27,8 @@ export const rootLoggerServiceFactory = createServiceFactory({ service: coreServices.rootLogger, deps: { config: coreServices.rootConfig, - schemaDiscovery: schemaDiscoveryServiceRef, }, - async factory({ config, schemaDiscovery }) { + async factory({ config }) { const logger = WinstonLogger.create({ meta: { service: 'backstage', @@ -43,12 +41,7 @@ export const rootLoggerServiceFactory = createServiceFactory({ transports: [new transports.Console()], }); - const secretEnumerator = await createConfigSecretEnumerator({ - logger, - additionalSchemas: ( - await schemaDiscovery?.getAdditionalSchemas() - )?.schemas, - }); + const secretEnumerator = await createConfigSecretEnumerator({ logger }); logger.addRedactions(secretEnumerator(config)); config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 607eaa010e..e4a1ecfdf1 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -32,7 +32,6 @@ import { GitLabIntegration } from '@backstage/integration'; import { HostDiscovery as HostDiscovery_2 } from '@backstage/backend-app-api'; import { IdentityService } from '@backstage/backend-plugin-api'; import { isChildPath } from '@backstage/cli-common'; -import { JsonObject } from '@backstage/types'; import { Knex } from 'knex'; import knexFactory from 'knex'; import { KubeConfig } from '@kubernetes/client-node'; @@ -561,9 +560,6 @@ export function loadBackendConfig(options: { logger: LoggerService; remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; - additionalSchemas?: { - [context: string]: JsonObject; - }; argv: string[]; watch?: boolean; }): Promise; diff --git a/packages/backend-common/src/config.test.ts b/packages/backend-common/src/config.test.ts deleted file mode 100644 index 05d1cbc3ec..0000000000 --- a/packages/backend-common/src/config.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { transports } from 'winston'; -import { loadBackendConfig } from './config'; -import { getRootLogger } from './logging'; - -describe('config', () => { - describe('loadBackendConfig', () => { - const env = process.env; - afterEach(() => { - jest.resetModules(); - process.env = env; - }); - - it('also hide secret config values coming from additional schemas', async () => { - const additionalConfigs = [ - { - context: 'test', - data: { - secretValue: 'shouldBeHidden', - }, - }, - ]; - - const additionalSchemas = { - test: { - type: 'object', - properties: { - secretValue: { - type: 'string', - visibility: 'secret', - }, - }, - }, - }; - - const logs: string[] = []; - jest - .spyOn(transports.Console.prototype, 'log') - .mockImplementation((s: any, next: any) => { - logs.push(s.message); - if (next) { - next(); - } - return undefined; - }); - - process.env.LOG_LEVEL = 'info'; - const logger = getRootLogger(); - await loadBackendConfig({ - logger, - argv: [], - additionalConfigs, - additionalSchemas, - }); - logger.info('test'); - logger.info('shouldBeHidden'); - logger.end(); - await new Promise(resolve => { - logger.on('finish', resolve); - }); - - expect(logs[0]).toMatch( - /Found \d+ new secrets in config that will be redacted/, - ); - expect(logs[1]).toEqual('test'); - expect(logs[2]).toEqual('[REDACTED]'); - }, 30000); - }); -}); diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index ac89f3f4ec..b452d8db70 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -22,7 +22,6 @@ import { LoggerService } from '@backstage/backend-plugin-api'; import { AppConfig, Config } from '@backstage/config'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; -import { JsonObject } from '@backstage/types'; /** * Load configuration for a Backend. @@ -36,13 +35,11 @@ export async function loadBackendConfig(options: { // process.argv or any other overrides remote?: LoadConfigOptionsRemote; additionalConfigs?: AppConfig[]; - additionalSchemas?: { [context: string]: JsonObject }; argv: string[]; watch?: boolean; }): Promise { const secretEnumerator = await createConfigSecretEnumerator({ logger: options.logger, - additionalSchemas: options.additionalSchemas, }); const { config } = await newLoadBackendConfig(options); diff --git a/packages/backend-dynamic-feature-service/api-report.md b/packages/backend-dynamic-feature-service/api-report.md index 5a9e93f50b..64bffb468c 100644 --- a/packages/backend-dynamic-feature-service/api-report.md +++ b/packages/backend-dynamic-feature-service/api-report.md @@ -7,6 +7,7 @@ import { BackendFeature } from '@backstage/backend-plugin-api'; import { BackstagePackageJson } from '@backstage/cli-node'; import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; +import { ConfigSchema } from '@backstage/config-loader'; import { EventBroker } from '@backstage/plugin-events-node'; import { EventsBackend } from '@backstage/plugin-events-backend'; import { FeatureDiscoveryService } from '@backstage/backend-plugin-api/alpha'; @@ -23,8 +24,8 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { RootLoggerService } from '@backstage/backend-plugin-api'; import { Router } from 'express'; -import { SchemaDiscoveryService } from '@backstage/backend-plugin-api/alpha'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; import { TaskRunner } from '@backstage/backend-tasks'; @@ -117,10 +118,32 @@ export const dynamicPluginsFeatureDiscoveryServiceFactory: () => ServiceFactory< >; // @public (undocumented) -export interface DynamicPluginsSchemaDiscoveryOptions { +export const dynamicPluginsFrontendSchemas: () => BackendFeature; + +// @public (undocumented) +export const dynamicPluginsRootLoggerServiceFactory: () => ServiceFactory< + RootLoggerService, + 'root' +>; + +// @public (undocumented) +export interface DynamicPluginsSchemasOptions { schemaLocator?: (pluginPackage: ScannedPluginPackage) => string; } +// @public (undocumented) +export interface DynamicPluginsSchemasService { + // (undocumented) + addDynamicPluginsSchemas(configSchema: ConfigSchema): Promise<{ + schema: ConfigSchema; + }>; +} + +// @public (undocumented) +export const dynamicPluginsSchemasServiceFactory: ( + options?: DynamicPluginsSchemasOptions | undefined, +) => ServiceFactory; + // @public (undocumented) export const dynamicPluginsServiceFactory: ( options?: DynamicPluginsFactoryOptions | undefined, @@ -226,10 +249,5 @@ export interface ScannedPluginPackage { manifest: ScannedPluginManifest; } -// @public (undocumented) -export const schemaDiscoveryServiceFactory: ( - options?: DynamicPluginsSchemaDiscoveryOptions | undefined, -) => ServiceFactory; - // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-dynamic-feature-service/package.json b/packages/backend-dynamic-feature-service/package.json index bc99c2a692..8699d66b2a 100644 --- a/packages/backend-dynamic-feature-service/package.json +++ b/packages/backend-dynamic-feature-service/package.json @@ -12,6 +12,16 @@ "backstage": { "role": "node-library" }, + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/backend-dynamic-feature-service" + }, "keywords": [ "backstage" ], @@ -26,6 +36,7 @@ "start": "backstage-cli package start" }, "dependencies": { + "@backstage/backend-app-api": "workspace:^", "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", @@ -34,6 +45,7 @@ "@backstage/config": "workspace:^", "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", + "@backstage/plugin-app-node": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-events-backend": "workspace:^", @@ -44,20 +56,21 @@ "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", + "@manypkg/get-packages": "^1.1.3", "@types/express": "^4.17.6", "chokidar": "^3.5.3", "express": "^4.17.1", - "fs-extra": "^7.0.1", + "fs-extra": "10.1.0", "lodash": "^4.17.21", "winston": "^3.2.1" }, "devDependencies": { - "@backstage/backend-app-api": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "wait-for-expect": "^3.0.2" }, "files": [ - "dist" + "dist", + "config.d.ts" ] } diff --git a/packages/backend-dynamic-feature-service/src/index.ts b/packages/backend-dynamic-feature-service/src/index.ts index 5570d8ee29..3768146124 100644 --- a/packages/backend-dynamic-feature-service/src/index.ts +++ b/packages/backend-dynamic-feature-service/src/index.ts @@ -17,3 +17,4 @@ export * from './loader'; export * from './scanner'; export * from './manager'; +export * from './schemas'; diff --git a/packages/backend-dynamic-feature-service/src/scanner/index.ts b/packages/backend-dynamic-feature-service/src/scanner/index.ts index 22b476c143..c5574cc38e 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/index.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/index.ts @@ -15,5 +15,3 @@ */ export type { ScannedPluginManifest, ScannedPluginPackage } from './types'; -export type { DynamicPluginsSchemaDiscoveryOptions } from './plugin-scanner'; -export { schemaDiscoveryServiceFactory } from './plugin-scanner'; diff --git a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts index a9e11ca60b..af1144f019 100644 --- a/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts +++ b/packages/backend-dynamic-feature-service/src/scanner/plugin-scanner.ts @@ -22,15 +22,7 @@ import * as path from 'path'; import * as url from 'url'; import debounce from 'lodash/debounce'; import { PackagePlatform, PackageRoles } from '@backstage/cli-node'; -import { - LoggerService, - coreServices, - createServiceFactory, -} from '@backstage/backend-plugin-api'; -import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; -import { findPaths } from '@backstage/cli-common'; -import { gatherDynamicPluginsSchemas } from './schemas'; -import { JsonObject } from '@backstage/types'; +import { LoggerService } from '@backstage/backend-plugin-api'; export interface DynamicPluginScannerOptions { config: Config; @@ -323,72 +315,3 @@ export class PluginScanner { this.untrackChanges(); } } - -/** - * @public - */ -export interface DynamicPluginsSchemaDiscoveryOptions { - /** - * Function that returns the path to the Json schema file for a given scanned plugin package. - * The path is either absolute, or relative to the plugin package root directory. - * - * Default behavior is to look for the `dist/configSchema.json` relative path. - * - * @param pluginPackage - The scanned plugin package. - * @returns the absolute or plugin-relative path to the Json schema file. - */ - schemaLocator?: (pluginPackage: ScannedPluginPackage) => string; -} - -/** - * @public - */ -export const schemaDiscoveryServiceFactory = createServiceFactory( - (options?: DynamicPluginsSchemaDiscoveryOptions) => ({ - service: schemaDiscoveryServiceRef, - deps: { - config: coreServices.rootConfig, - }, - factory({ config }) { - let schemas: { [context: string]: JsonObject } | undefined; - - return { - async getAdditionalSchemas(): Promise<{ - schemas: { [context: string]: JsonObject }; - }> { - if (schemas) { - return { - schemas, - }; - } - - const logger = { - ...console, - child() { - return this; - }, - }; - - const scanner = PluginScanner.create({ - config, - logger, - // eslint-disable-next-line no-restricted-syntax - backstageRoot: findPaths(__dirname).targetRoot, - preferAlpha: true, - }); - - const { packages } = await scanner.scanRoot(); - - schemas = await gatherDynamicPluginsSchemas( - packages, - logger, - options?.schemaLocator, - ); - return { - schemas, - }; - }, - }; - }, - }), -); diff --git a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts b/packages/backend-dynamic-feature-service/src/scanner/schemas.ts deleted file mode 100644 index f00828859a..0000000000 --- a/packages/backend-dynamic-feature-service/src/scanner/schemas.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ScannedPluginPackage } from '@backstage/backend-dynamic-feature-service'; -import fs from 'fs-extra'; -import * as path from 'path'; -import * as url from 'url'; -import { isEmpty } from 'lodash'; -import { LoggerService } from '@backstage/backend-plugin-api'; -import { JsonObject } from '@backstage/types'; - -export async function gatherDynamicPluginsSchemas( - packages: ScannedPluginPackage[], - logger: LoggerService, - schemaLocator: (pluginPackage: ScannedPluginPackage) => string = () => - path.join('dist', 'configSchema.json'), -): Promise<{ [context: string]: JsonObject }> { - const allSchemas: { [context: string]: JsonObject } = {}; - - for (const pluginPackage of packages) { - let schemaLocation = schemaLocator(pluginPackage); - - if (!path.isAbsolute(schemaLocation)) { - let pluginLocation = url.fileURLToPath(pluginPackage.location); - if (path.basename(pluginLocation) === 'alpha') { - pluginLocation = path.dirname(pluginLocation); - } - schemaLocation = path.resolve(pluginLocation, schemaLocation); - } - - if (!(await fs.pathExists(schemaLocation))) { - continue; - } - - const serialized = await fs.readJson(schemaLocation); - if (!serialized) { - continue; - } - - if (isEmpty(serialized)) { - continue; - } - - if (!serialized?.$schema || serialized?.type !== 'object') { - logger.error( - `Serialized configuration schema is invalid for plugin ${pluginPackage.manifest.name}`, - ); - continue; - } - - allSchemas[schemaLocation] = serialized; - } - - return allSchemas; -} diff --git a/packages/backend-dynamic-feature-service/src/schemas/appBackendModule.ts b/packages/backend-dynamic-feature-service/src/schemas/appBackendModule.ts new file mode 100644 index 0000000000..b44913edfd --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/schemas/appBackendModule.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { dynamicPluginsSchemasServiceRef } from './schemas'; +import { + configSchemaExtensionPoint, + loadCompiledConfigSchema, +} from '@backstage/plugin-app-node'; +import { resolvePackagePath } from '@backstage/backend-common'; + +/** @public */ +export const dynamicPluginsFrontendSchemas = createBackendModule({ + pluginId: 'app', + moduleId: 'core.dynamicplugins.frontendSchemas', + register(reg) { + reg.registerInit({ + deps: { + config: coreServices.rootConfig, + schemas: dynamicPluginsSchemasServiceRef, + configSchemaExtension: configSchemaExtensionPoint, + }, + async init({ config, schemas, configSchemaExtension }) { + const appPackageName = + config.getOptionalString('app.packageName') ?? 'app'; + const appDistDir = resolvePackagePath(appPackageName, 'dist'); + const compiledConfigSchema = await loadCompiledConfigSchema(appDistDir); + if (compiledConfigSchema) { + configSchemaExtension.setConfigSchema( + (await schemas.addDynamicPluginsSchemas(compiledConfigSchema)) + .schema, + ); + } + }, + }); + }, +}); diff --git a/packages/backend-dynamic-feature-service/src/schemas/index.ts b/packages/backend-dynamic-feature-service/src/schemas/index.ts new file mode 100644 index 0000000000..94c167679b --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/schemas/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + dynamicPluginsSchemasServiceFactory, + type DynamicPluginsSchemasService, + type DynamicPluginsSchemasOptions, +} from './schemas'; + +export { dynamicPluginsFrontendSchemas } from './appBackendModule'; +export { dynamicPluginsRootLoggerServiceFactory } from './rootLoggerServiceFactory'; diff --git a/packages/backend-dynamic-feature-service/src/schemas/rootLoggerServiceFactory.ts b/packages/backend-dynamic-feature-service/src/schemas/rootLoggerServiceFactory.ts new file mode 100644 index 0000000000..6681a65a75 --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/schemas/rootLoggerServiceFactory.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createServiceFactory, + coreServices, +} from '@backstage/backend-plugin-api'; +import { WinstonLogger } from '@backstage/backend-app-api'; +import { transports, format } from 'winston'; +import { createConfigSecretEnumerator } from '@backstage/backend-app-api'; +import { loadConfigSchema } from '@backstage/config-loader'; +import { getPackages } from '@manypkg/get-packages'; +import { dynamicPluginsSchemasServiceRef } from './schemas'; + +/** @public */ +export const dynamicPluginsRootLoggerServiceFactory = createServiceFactory({ + service: coreServices.rootLogger, + deps: { + config: coreServices.rootConfig, + schemas: dynamicPluginsSchemasServiceRef, + }, + async factory({ config, schemas }) { + const logger = WinstonLogger.create({ + meta: { + service: 'backstage', + }, + level: process.env.LOG_LEVEL || 'info', + format: + process.env.NODE_ENV === 'production' + ? format.json() + : WinstonLogger.colorFormat(), + transports: [new transports.Console()], + }); + + const configSchema = await loadConfigSchema({ + dependencies: ( + await getPackages(process.cwd()) + ).packages.map(p => p.packageJson.name), + }); + + const secretEnumerator = await createConfigSecretEnumerator({ + logger, + schema: (await schemas.addDynamicPluginsSchemas(configSchema)).schema, + }); + logger.addRedactions(secretEnumerator(config)); + config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); + + return logger; + }, +}); diff --git a/packages/backend-dynamic-feature-service/src/schemas/schemas.ts b/packages/backend-dynamic-feature-service/src/schemas/schemas.ts new file mode 100644 index 0000000000..0c03e93ada --- /dev/null +++ b/packages/backend-dynamic-feature-service/src/schemas/schemas.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ScannedPluginPackage } from '../scanner/types'; +import { + coreServices, + createServiceFactory, + createServiceRef, +} from '@backstage/backend-plugin-api'; +import { findPaths } from '@backstage/cli-common'; + +import fs from 'fs-extra'; +import * as path from 'path'; +import * as url from 'url'; +import { isEmpty } from 'lodash'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { JsonObject } from '@backstage/types'; +import { PluginScanner } from '../scanner/plugin-scanner'; +import { ConfigSchema, loadConfigSchema } from '@backstage/config-loader'; + +/** + * + * @public + * */ +export interface DynamicPluginsSchemasService { + addDynamicPluginsSchemas(configSchema: ConfigSchema): Promise<{ + schema: ConfigSchema; + }>; +} + +/** + * A service that provides the config schemas of scanned dynamic plugins. + * + * @public + */ +export const dynamicPluginsSchemasServiceRef = + createServiceRef({ + id: 'core.dynamicplugins.schemas', + scope: 'root', + }); + +/** + * @public + */ +export interface DynamicPluginsSchemasOptions { + /** + * Function that returns the path to the Json schema file for a given scanned plugin package. + * The path is either absolute, or relative to the plugin package root directory. + * + * Default behavior is to look for the `dist/configSchema.json` relative path. + * + * @param pluginPackage - The scanned plugin package. + * @returns the absolute or plugin-relative path to the Json schema file. + */ + schemaLocator?: (pluginPackage: ScannedPluginPackage) => string; +} + +/** + * @public + */ +export const dynamicPluginsSchemasServiceFactory = createServiceFactory( + (options?: DynamicPluginsSchemasOptions) => ({ + service: dynamicPluginsSchemasServiceRef, + deps: { + config: coreServices.rootConfig, + }, + factory({ config }) { + let additionalSchemas: { [context: string]: JsonObject } | undefined; + + return { + async addDynamicPluginsSchemas(configSchema: ConfigSchema): Promise<{ + schema: ConfigSchema; + }> { + if (!additionalSchemas) { + const logger = { + ...console, + child() { + return this; + }, + }; + + const scanner = PluginScanner.create({ + config, + logger, + // eslint-disable-next-line no-restricted-syntax + backstageRoot: findPaths(__dirname).targetRoot, + preferAlpha: true, + }); + + const { packages } = await scanner.scanRoot(); + + additionalSchemas = await gatherDynamicPluginsSchemas( + packages, + logger, + options?.schemaLocator, + ); + } + + const serialized = configSchema.serialize(); + if (serialized?.backstageConfigSchemaVersion !== 1) { + throw new Error( + 'Serialized configuration schema is invalid or has an invalid version number', + ); + } + const schemas = serialized.schemas as { + value: JsonObject; + path: string; + }[]; + + schemas.push( + ...Object.keys(additionalSchemas).map(context => { + return { + path: context, + value: additionalSchemas![context], + }; + }), + ); + serialized.schemas = schemas; + return { + schema: await loadConfigSchema({ + serialized, + }), + }; + }, + }; + }, + }), +); + +/** @internal */ +async function gatherDynamicPluginsSchemas( + packages: ScannedPluginPackage[], + logger: LoggerService, + schemaLocator: (pluginPackage: ScannedPluginPackage) => string = () => + path.join('dist', 'configSchema.json'), +): Promise<{ [context: string]: JsonObject }> { + const allSchemas: { [context: string]: JsonObject } = {}; + + for (const pluginPackage of packages) { + let schemaLocation = schemaLocator(pluginPackage); + + if (!path.isAbsolute(schemaLocation)) { + let pluginLocation = url.fileURLToPath(pluginPackage.location); + if (path.basename(pluginLocation) === 'alpha') { + pluginLocation = path.dirname(pluginLocation); + } + schemaLocation = path.resolve(pluginLocation, schemaLocation); + } + + if (!(await fs.pathExists(schemaLocation))) { + continue; + } + + const serialized = await fs.readJson(schemaLocation); + if (!serialized) { + continue; + } + + if (isEmpty(serialized)) { + continue; + } + + if (!serialized?.$schema || serialized?.type !== 'object') { + logger.error( + `Serialized configuration schema is invalid for plugin ${pluginPackage.manifest.name}`, + ); + continue; + } + + allSchemas[schemaLocation] = serialized; + } + + return allSchemas; +} diff --git a/packages/backend-plugin-api/api-report-alpha.md b/packages/backend-plugin-api/api-report-alpha.md index 0d9eaf4a0e..81b378a671 100644 --- a/packages/backend-plugin-api/api-report-alpha.md +++ b/packages/backend-plugin-api/api-report-alpha.md @@ -4,7 +4,6 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { JsonObject } from '@backstage/types'; import { ServiceRef } from '@backstage/backend-plugin-api'; // @alpha (undocumented) @@ -21,21 +20,5 @@ export const featureDiscoveryServiceRef: ServiceRef< 'root' >; -// @alpha (undocumented) -export interface SchemaDiscoveryService { - // (undocumented) - getAdditionalSchemas(): Promise<{ - schemas: { - [context: string]: JsonObject; - }; - }>; -} - -// @alpha -export const schemaDiscoveryServiceRef: ServiceRef< - SchemaDiscoveryService, - 'root' ->; - // (No @packageDocumentation comment for this package) ``` diff --git a/packages/backend-plugin-api/package.json b/packages/backend-plugin-api/package.json index 5476a75cf3..a2946bd803 100644 --- a/packages/backend-plugin-api/package.json +++ b/packages/backend-plugin-api/package.json @@ -47,7 +47,6 @@ "dependencies": { "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", - "@backstage/config-loader": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/types": "workspace:^", diff --git a/packages/backend-plugin-api/src/alpha.ts b/packages/backend-plugin-api/src/alpha.ts index 7d7e3177c2..baee739f49 100644 --- a/packages/backend-plugin-api/src/alpha.ts +++ b/packages/backend-plugin-api/src/alpha.ts @@ -16,11 +16,8 @@ import { BackendFeature, - coreServices, - createServiceFactory, createServiceRef, } from '@backstage/backend-plugin-api'; -import { JsonObject } from '@backstage/types'; /** @alpha */ export interface FeatureDiscoveryService { @@ -36,34 +33,3 @@ export const featureDiscoveryServiceRef = id: 'core.featureDiscovery', scope: 'root', }); - -/** @alpha */ -export interface SchemaDiscoveryService { - getAdditionalSchemas(): Promise<{ - schemas: { [context: string]: JsonObject }; - }>; -} - -/** - * An optional service that can be used to dynamically load in additional BackendFeatures at runtime. - * @alpha - */ -export const schemaDiscoveryServiceRef = - createServiceRef({ - id: 'core.schemaDiscovery', - scope: 'root', - defaultFactory: async service => - createServiceFactory({ - service, - deps: { - config: coreServices.rootConfig, - }, - factory() { - return { - async getAdditionalSchemas() { - return { schemas: {} }; - }, - }; - }, - }), - }); diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index c3fc9f522e..66572dd960 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -193,9 +193,6 @@ export type LoadConfigSchemaOptions = ( } ) & { noUndeclaredProperties?: boolean; - additionalSchemas?: { - [context: string]: JsonObject; - }; }; // @public diff --git a/packages/config-loader/src/schema/index.ts b/packages/config-loader/src/schema/index.ts index 01c18ee905..1dcb9d7b4b 100644 --- a/packages/config-loader/src/schema/index.ts +++ b/packages/config-loader/src/schema/index.ts @@ -22,5 +22,4 @@ export type { ConfigVisibility, ConfigSchemaProcessingOptions, TransformFunc, - ConfigSchemaPackageEntry, } from './types'; diff --git a/packages/config-loader/src/schema/load.test.ts b/packages/config-loader/src/schema/load.test.ts index f9299340c3..525565f3d5 100644 --- a/packages/config-loader/src/schema/load.test.ts +++ b/packages/config-loader/src/schema/load.test.ts @@ -126,57 +126,6 @@ describe('loadConfigSchema', () => { ); }); - it('should include additional schemas', async () => { - const schema = await loadConfigSchema({ - serialized: { - backstageConfigSchemaVersion: 1, - schemas: [ - { - path: 'mainSchema', - value: { - type: 'object', - properties: { - key1: { type: 'string', visibility: 'frontend' }, - }, - }, - }, - ], - }, - additionalSchemas: { - additionalSchema: { - type: 'object', - properties: { - additionalKey: { type: 'string', visibility: 'frontend' }, - }, - }, - }, - }); - - expect(schema.serialize()).toEqual({ - backstageConfigSchemaVersion: 1, - schemas: [ - { - path: 'mainSchema', - value: { - type: 'object', - properties: { - key1: { type: 'string', visibility: 'frontend' }, - }, - }, - }, - { - path: 'additionalSchema', - value: { - type: 'object', - properties: { - additionalKey: { type: 'string', visibility: 'frontend' }, - }, - }, - }, - ], - }); - }); - describe('should consider schema', () => { it('when filtering simple config', async () => { mockDir.setContent({ diff --git a/packages/config-loader/src/schema/load.ts b/packages/config-loader/src/schema/load.ts index 03b85b2756..885da44116 100644 --- a/packages/config-loader/src/schema/load.ts +++ b/packages/config-loader/src/schema/load.ts @@ -43,7 +43,6 @@ export type LoadConfigSchemaOptions = } ) & { noUndeclaredProperties?: boolean; - additionalSchemas?: { [context: string]: JsonObject }; }; function errorsToError(errors: ValidationError[]): Error { @@ -84,16 +83,6 @@ export async function loadConfigSchema( } schemas = serialized.schemas as ConfigSchemaPackageEntry[]; } - if (options.additionalSchemas) { - schemas.push( - ...Object.keys(options.additionalSchemas).map(context => { - return { - path: context, - value: options.additionalSchemas![context], - }; - }), - ); - } const validate = compileConfigSchemas(schemas, { noUndeclaredProperties: options.noUndeclaredProperties, diff --git a/plugins/app-backend/api-report.md b/plugins/app-backend/api-report.md index fd85b68a87..0afc7021ad 100644 --- a/plugins/app-backend/api-report.md +++ b/plugins/app-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { Config } from '@backstage/config'; +import { ConfigSchema } from '@backstage/config-loader'; import express from 'express'; -import { JsonObject } from '@backstage/types'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -14,9 +14,6 @@ export function createRouter(options: RouterOptions): Promise; // @public (undocumented) export interface RouterOptions { - additionalSchemas?: { - [context: string]: JsonObject; - }; appPackageName: string; // (undocumented) config: Config; @@ -24,6 +21,7 @@ export interface RouterOptions { disableConfigInjection?: boolean; // (undocumented) logger: Logger; + schema?: ConfigSchema; staticFallbackHandler?: express.Handler; } ``` diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index dc041a2593..f09ba5c5ec 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -19,7 +19,11 @@ import { resolve as resolvePath } from 'path'; import { Logger } from 'winston'; import { AppConfig, Config } from '@backstage/config'; import { JsonObject } from '@backstage/types'; -import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; +import { + ConfigSchema, + loadConfigSchema, + readEnvConfig, +} from '@backstage/config-loader'; type InjectOptions = { appConfigs: AppConfig[]; @@ -74,7 +78,7 @@ type ReadOptions = { env: { [name: string]: string | undefined }; appDistDir: string; config: Config; - additionalSchemas?: { [context: string]: JsonObject }; + schema?: ConfigSchema; }; /** @@ -91,10 +95,11 @@ export async function readConfigs(options: ReadOptions): Promise { const serializedSchema = await fs.readJson(schemaPath); try { - const schema = await loadConfigSchema({ - serialized: serializedSchema, - additionalSchemas: options.additionalSchemas, - }); + const schema = + options.schema || + (await loadConfigSchema({ + serialized: serializedSchema, + })); const frontendConfigs = await schema.process( [{ data: config.get() as JsonObject, context: 'app' }], diff --git a/plugins/app-backend/src/service/appPlugin.ts b/plugins/app-backend/src/service/appPlugin.ts index 07616e832f..a2ea05623f 100644 --- a/plugins/app-backend/src/service/appPlugin.ts +++ b/plugins/app-backend/src/service/appPlugin.ts @@ -19,10 +19,13 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; -import { schemaDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; import { createRouter } from './router'; import { loggerToWinstonLogger } from '@backstage/backend-common'; -import { staticFallbackHandlerExtensionPoint } from '@backstage/plugin-app-node'; +import { + configSchemaExtensionPoint, + staticFallbackHandlerExtensionPoint, +} from '@backstage/plugin-app-node'; +import { ConfigSchema } from '@backstage/config-loader'; /** * The App plugin is responsible for serving the frontend app bundle and static assets. @@ -32,6 +35,7 @@ export const appPlugin = createBackendPlugin({ pluginId: 'app', register(env) { let staticFallbackHandler: express.Handler | undefined; + let schema: ConfigSchema | undefined; env.registerExtensionPoint(staticFallbackHandlerExtensionPoint, { setStaticFallbackHandler(handler) { @@ -44,17 +48,28 @@ export const appPlugin = createBackendPlugin({ }, }); + env.registerExtensionPoint(configSchemaExtensionPoint, { + setConfigSchema(configSchema) { + if (schema) { + throw new Error( + 'Attempted to set config schema for the app-backend twice', + ); + } + schema = configSchema; + }, + }); + env.registerInit({ deps: { logger: coreServices.logger, config: coreServices.rootConfig, database: coreServices.database, httpRouter: coreServices.httpRouter, - schemaDiscovery: schemaDiscoveryServiceRef, }, - async init({ logger, config, database, httpRouter, schemaDiscovery }) { + async init({ logger, config, database, httpRouter }) { const appPackageName = config.getOptionalString('app.packageName') ?? 'app'; + const winstonLogger = loggerToWinstonLogger(logger); const router = await createRouter({ @@ -63,9 +78,7 @@ export const appPlugin = createBackendPlugin({ database, appPackageName, staticFallbackHandler, - additionalSchemas: ( - await schemaDiscovery?.getAdditionalSchemas() - )?.schemas, + schema, }); httpRouter.use(router); }, diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index 204d5df26a..17c5fd4f84 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -37,7 +37,7 @@ import { CACHE_CONTROL_NO_CACHE, CACHE_CONTROL_REVALIDATE_CACHE, } from '../lib/headers'; -import { JsonObject } from '@backstage/types'; +import { ConfigSchema } from '@backstage/config-loader'; // express uses mime v1 while we only have types for mime v2 type Mime = { lookup(arg0: string): string }; @@ -89,14 +89,10 @@ export interface RouterOptions { /** * - * Provides a map of additional config schemas, in addition to the serialized schemas - * generated during the application build. - * This is useful when additional plugins are dynamically loaded in the application at start, - * which were not part of the application build. This option allows feeding the corresponding - * JSON schemas. + * Provides a ConfigSchema. * */ - additionalSchemas?: { [context: string]: JsonObject }; + schema?: ConfigSchema; } /** @public */ @@ -133,7 +129,7 @@ export async function createRouter( config, appDistDir, env: process.env, - additionalSchemas: options.additionalSchemas, + schema: options.schema, }); injectedConfigPath = await injectConfig({ appConfigs, logger, staticDir }); diff --git a/plugins/app-node/api-report.md b/plugins/app-node/api-report.md index 504c9d48c1..58e51e2b5c 100644 --- a/plugins/app-node/api-report.md +++ b/plugins/app-node/api-report.md @@ -3,9 +3,23 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { ConfigSchema } from '@backstage/config-loader'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; +// @public +export interface ConfigSchemaExtensionPoint { + setConfigSchema(configSchema: ConfigSchema): void; +} + +// @public +export const configSchemaExtensionPoint: ExtensionPoint; + +// @public +export function loadCompiledConfigSchema( + appDistDir: string, +): Promise; + // @public export interface StaticFallbackHandlerExtensionPoint { setStaticFallbackHandler(handler: Handler): void; diff --git a/plugins/app-node/package.json b/plugins/app-node/package.json index 8da14adf64..d55fd05e98 100644 --- a/plugins/app-node/package.json +++ b/plugins/app-node/package.json @@ -29,7 +29,9 @@ ], "dependencies": { "@backstage/backend-plugin-api": "workspace:^", + "@backstage/config-loader": "workspace:^", "@types/express": "^4.17.6", - "express": "^4.17.1" + "express": "^4.17.1", + "fs-extra": "10.1.0" } } diff --git a/plugins/app-node/src/extensions.ts b/plugins/app-node/src/extensions.ts index 6a75a5b791..6cb612f8df 100644 --- a/plugins/app-node/src/extensions.ts +++ b/plugins/app-node/src/extensions.ts @@ -15,6 +15,7 @@ */ import { createExtensionPoint } from '@backstage/backend-plugin-api'; +import { ConfigSchema } from '@backstage/config-loader'; import { Handler } from 'express'; /** @@ -50,3 +51,25 @@ export const staticFallbackHandlerExtensionPoint = createExtensionPoint({ id: 'app.staticFallbackHandler', }); + +/** + * The interface for {@link configSchemaExtensionPoint}. + * + * @public + */ +export interface ConfigSchemaExtensionPoint { + /** + * Sets the config schema. This can only be done once. + */ + setConfigSchema(configSchema: ConfigSchema): void; +} + +/** + * An extension point the exposes the ability to override the config schema used by the frontend application. + * + * @public + */ +export const configSchemaExtensionPoint = + createExtensionPoint({ + id: 'app.configSchema', + }); diff --git a/plugins/app-node/src/index.ts b/plugins/app-node/src/index.ts index c5189fa8b9..4695471cf8 100644 --- a/plugins/app-node/src/index.ts +++ b/plugins/app-node/src/index.ts @@ -23,4 +23,8 @@ export { staticFallbackHandlerExtensionPoint, type StaticFallbackHandlerExtensionPoint, + configSchemaExtensionPoint, + type ConfigSchemaExtensionPoint, } from './extensions'; + +export { loadCompiledConfigSchema } from './schema'; diff --git a/plugins/app-node/src/schema.ts b/plugins/app-node/src/schema.ts new file mode 100644 index 0000000000..a38fd585d9 --- /dev/null +++ b/plugins/app-node/src/schema.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'path'; +import { ConfigSchema, loadConfigSchema } from '@backstage/config-loader'; + +/** + * Loads the config schema that is embedded in the frontend build. + * + * @public + */ +export async function loadCompiledConfigSchema( + appDistDir: string, +): Promise { + const schemaPath = resolvePath(appDistDir, '.config-schema.json'); + if (await fs.pathExists(schemaPath)) { + const serializedSchema = await fs.readJson(schemaPath); + + return await loadConfigSchema({ + serialized: serializedSchema, + }); + } + + return undefined; +} diff --git a/yarn.lock b/yarn.lock index 0e32adda8c..3e3843b1a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3382,6 +3382,7 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" + "@backstage/plugin-app-node": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-events-backend": "workspace:^" @@ -3392,10 +3393,11 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" + "@manypkg/get-packages": ^1.1.3 "@types/express": ^4.17.6 chokidar: ^3.5.3 express: ^4.17.1 - fs-extra: ^7.0.1 + fs-extra: 10.1.0 lodash: ^4.17.21 wait-for-expect: ^3.0.2 winston: ^3.2.1 @@ -3429,7 +3431,6 @@ __metadata: "@backstage/backend-tasks": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" - "@backstage/config-loader": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/types": "workspace:^" @@ -4647,8 +4648,10 @@ __metadata: dependencies: "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" + "@backstage/config-loader": "workspace:^" "@types/express": ^4.17.6 express: ^4.17.1 + fs-extra: 10.1.0 languageName: unknown linkType: soft From 23f447c0ed472f843b80f779c5f873ffa5a12417 Mon Sep 17 00:00:00 2001 From: Matt Wise Date: Sun, 11 Feb 2024 23:19:14 -0600 Subject: [PATCH 28/34] fix s3 downloads resulting in flat folder structure Signed-off-by: Matt Wise --- packages/backend-common/src/reading/AwsS3UrlReader.ts | 3 ++- .../src/reading/tree/ReadableArrayResponse.ts | 11 ++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 23ef0f7b27..3ba4eebb48 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -46,6 +46,7 @@ import { import { AbortController } from '@aws-sdk/abort-controller'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { Readable } from 'stream'; +import { relative } from 'path/posix'; export const DEFAULT_REGION = 'us-east-1'; @@ -345,7 +346,7 @@ export class AwsS3UrlReader implements UrlReader { responses.push({ data: s3ObjectData, - path: String(allObjects[i]), + path: relative(path, String(allObjects[i])), lastModifiedAt: response?.LastModified ?? undefined, }); } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index 1429931142..3c8bfd1fbc 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -15,7 +15,7 @@ */ import concatStream from 'concat-stream'; -import platformPath, { basename } from 'path'; +import platformPath, { dirname } from 'path'; import getRawBody from 'raw-body'; import fs from 'fs-extra'; @@ -96,12 +96,9 @@ export class ReadableArrayResponse implements ReadTreeResponse { for (let i = 0; i < this.stream.length; i++) { if (!this.stream[i].path.endsWith('/')) { - await pipeline( - this.stream[i].data, - fs.createWriteStream( - platformPath.join(dir, basename(this.stream[i].path)), - ), - ); + const filePath = platformPath.join(dir, this.stream[i].path); + await fs.mkdir(dirname(filePath), { recursive: true }); + await pipeline(this.stream[i].data, fs.createWriteStream(filePath)); } } From 497382cc184633729e7c71e3bfd960257f344b83 Mon Sep 17 00:00:00 2001 From: Matt Wise Date: Mon, 12 Feb 2024 01:20:46 -0600 Subject: [PATCH 29/34] update unit test for ReadableArrayResponse to handle subdir/relativePath Signed-off-by: Matt Wise --- .../src/reading/tree/ReadableArrayResponse.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index 0a27bdf09c..e2bfe1af59 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -19,6 +19,7 @@ import path from 'path'; import { FromReadableArrayOptions } from '../types'; import { ReadableArrayResponse } from './ReadableArrayResponse'; import { createMockDirectory } from '@backstage/backend-test-utils'; +import { relative } from 'path/posix'; const name1 = 'file1.yaml'; const file1 = fs.readFileSync( @@ -74,9 +75,12 @@ describe('ReadableArrayResponse', () => { }); it('should extract entire archive into directory', async () => { + const relativePath1 = relative(sourceDir.path, path1); + const relativePath2 = relative(sourceDir.path, path2); + const arr: FromReadableArrayOptions = [ - { data: createReadStream(path1), path: path1 }, - { data: createReadStream(path2), path: path2 }, + { data: createReadStream(path1), path: relativePath1 }, + { data: createReadStream(path2), path: relativePath2 }, ]; const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); From b0f33dc77a33c0e6187c9dae594cdacf444df6cb Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 12 Feb 2024 20:11:23 +0100 Subject: [PATCH 30/34] Allow `backend-dynamic-feature-service` to depend on `backend-app-api` Signed-off-by: David Festal --- scripts/verify-local-dependencies.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/verify-local-dependencies.js b/scripts/verify-local-dependencies.js index 5d75348681..e2aaf8c33a 100755 --- a/scripts/verify-local-dependencies.js +++ b/scripts/verify-local-dependencies.js @@ -90,6 +90,7 @@ const roleRules = [ '@backstage/backend-common', '@backstage/backend-defaults', '@backstage/backend-test-utils', + '@backstage/backend-dynamic-feature-service', ], message: `Plugin package SOURCE_NAME with role SOURCE_ROLE has a runtime dependency on package TARGET_NAME, which is not permitted. If you are using this dependency for dev server purposes, you can move it to devDependencies instead.`, }, From 5d338bea6ec90d5a3d9dda784a4921e5581081b1 Mon Sep 17 00:00:00 2001 From: David Festal Date: Mon, 12 Feb 2024 20:20:38 +0100 Subject: [PATCH 31/34] fix a `package.json` Signed-off-by: David Festal --- packages/backend-dynamic-feature-service/package.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/backend-dynamic-feature-service/package.json b/packages/backend-dynamic-feature-service/package.json index 8699d66b2a..7066c6c2ea 100644 --- a/packages/backend-dynamic-feature-service/package.json +++ b/packages/backend-dynamic-feature-service/package.json @@ -5,9 +5,7 @@ "main": "src/index.ts", "types": "src/index.ts", "publishConfig": { - "access": "public", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "access": "public" }, "backstage": { "role": "node-library" @@ -16,6 +14,13 @@ ".": "./src/index.ts", "./package.json": "./package.json" }, + "typesVersions": { + "*": { + "package.json": [ + "package.json" + ] + } + }, "homepage": "https://backstage.io", "repository": { "type": "git", From 842171ababafb851264349cf3540dd6e0e2c7d6f Mon Sep 17 00:00:00 2001 From: Matt Wise Date: Mon, 12 Feb 2024 15:35:15 -0600 Subject: [PATCH 32/34] add changeset Signed-off-by: Matt Wise --- .changeset/brown-peaches-hunt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/brown-peaches-hunt.md diff --git a/.changeset/brown-peaches-hunt.md b/.changeset/brown-peaches-hunt.md new file mode 100644 index 0000000000..fc10acc211 --- /dev/null +++ b/.changeset/brown-peaches-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Fix a bug with S3 Fetch that caused all objects to be flattened within a single folder on the local file system. From f02f638bc30a3e0aa6e233b319faac7119fbf6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 13 Feb 2024 11:31:40 +0100 Subject: [PATCH 33/34] Update plugins/devtools/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/devtools/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/devtools/README.md b/plugins/devtools/README.md index 2d21e8e6f2..ca9b436acc 100644 --- a/plugins/devtools/README.md +++ b/plugins/devtools/README.md @@ -418,6 +418,8 @@ By default, only packages with names starting with `@backstage` and `@internal` devTools: info: packagePrefixes: + # Note that you MUST have quotes around these. The YAML won't be valid + # if you don't, because of the leading at-symbols. - '@roadiehq/backstage-' - '@spotify/backstage-' ``` From 8b62f68b48e4cebf5dcdda4da9a3fa05bdfc7fd4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 13 Feb 2024 11:46:25 +0100 Subject: [PATCH 34/34] Apply suggestions from code review Signed-off-by: Patrik Oldsberg --- .changeset/late-poets-love.md | 2 +- .changeset/rich-bees-fry.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/late-poets-love.md b/.changeset/late-poets-love.md index bd6692560d..03b975b2b5 100644 --- a/.changeset/late-poets-love.md +++ b/.changeset/late-poets-love.md @@ -2,4 +2,4 @@ '@backstage/backend-app-api': patch --- -Allow the `createConfigSecretEnumerator` to take an optional `configSchema` argument with an already-loaded global configuration schema. +Allow the `createConfigSecretEnumerator` to take an optional `schema` argument with an already-loaded global configuration schema. diff --git a/.changeset/rich-bees-fry.md b/.changeset/rich-bees-fry.md index 3dcfa95532..281719340b 100644 --- a/.changeset/rich-bees-fry.md +++ b/.changeset/rich-bees-fry.md @@ -1,6 +1,6 @@ --- -'@backstage/plugin-app-backend': minor -'@backstage/plugin-app-node': minor +'@backstage/plugin-app-backend': patch +'@backstage/plugin-app-node': patch --- Allow the `app-backend` plugin to use a global configuration schema provided externally through an extension.