diff --git a/packages/backend-app-api/src/config/config.test.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.test.ts similarity index 98% rename from packages/backend-app-api/src/config/config.test.ts rename to packages/backend-app-api/src/config/ObservableConfigProxy.test.ts index 67db6996cd..e90225db1a 100644 --- a/packages/backend-app-api/src/config/config.test.ts +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.test.ts @@ -16,7 +16,7 @@ import { LoggerService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; -import { ObservableConfigProxy } from './config'; +import { ObservableConfigProxy } from './ObservableConfigProxy'; describe('ObservableConfigProxy', () => { const errLogger = { diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index 0f07be85d2..6adebbdf52 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -21,47 +21,44 @@ import { findPaths } from '@backstage/cli-common'; import { loadConfigSchema, loadConfig, - ConfigSchema, ConfigTarget, LoadConfigOptionsRemote, } from '@backstage/config-loader'; -import { AppConfig, Config, ConfigReader } from '@backstage/config'; +import { Config, ConfigReader } from '@backstage/config'; import { getPackages } from '@manypkg/get-packages'; import { ObservableConfigProxy } from './ObservableConfigProxy'; import { isValidUrl } from '../lib/urls'; -import { setRootLoggerRedactionList } from './logging/rootLogger'; - -// Fetch the schema and get all the secrets to pass to the rootLogger for redaction -const updateRedactionList = ( - schema: ConfigSchema, - configs: AppConfig[], - logger: LoggerService, -) => { - const secretAppConfigs = schema.process(configs, { - visibility: ['secret'], - ignoreSchemaErrors: true, +/** @public */ +export async function createConfigSecretEnumerator(options: { + logger: LoggerService; + dir?: string; +}): Promise<(config: Config) => Iterable> { + const { logger, dir = process.cwd() } = options; + const { packages } = await getPackages(dir); + const schema = await loadConfigSchema({ + dependencies: packages.map(p => p.packageJson.name), }); - const secretConfig = ConfigReader.fromConfigs(secretAppConfigs); - const values = new Set(); - const data = secretConfig.get(); - JSON.parse( - JSON.stringify(data), - (_, v) => typeof v === 'string' && values.add(v), - ); - - logger.info( - `${values.size} secret${ - values.size > 1 ? 's' : '' - } found in the config which will be redacted`, - ); - - setRootLoggerRedactionList(Array.from(values)); -}; - -// A global used to ensure that only a single file watcher is active at a time. -let currentCancelFunc: () => void; + return (config: Config) => { + const [secretsData] = schema.process( + [{ data: config.get(), context: 'schema-enumerator' }], + { + visibility: ['secret'], + ignoreSchemaErrors: true, + }, + ); + const secrets = new Set(); + JSON.parse( + JSON.stringify(secretsData), + (_, v) => typeof v === 'string' && secrets.add(v), + ); + logger.info( + `Found ${secrets.size} new secrets in config that will be redacted`, + ); + return secrets; + }; +} /** * Load configuration for a Backend. @@ -75,7 +72,7 @@ export async function loadBackendConfig(options: { // process.argv or any other overrides remote?: LoadConfigOptionsRemote; argv: string[]; -}): Promise { +}): Promise<{ config: Config }> { const args = parseArgs(options.argv); const configTargets: ConfigTarget[] = [args.config ?? []] @@ -85,13 +82,7 @@ export async function loadBackendConfig(options: { /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); - // TODO(hhogg): This is fetching _all_ of the packages of the monorepo - // in order to find the secrets for redactions, however we only care about - // the backend ones, we need to find a way to exclude the frontend packages. - const { packages } = await getPackages(paths.targetDir); - const schema = await loadConfigSchema({ - dependencies: packages.map(p => p.packageJson.name), - }); + let currentCancelFunc: (() => void) | undefined = undefined; const config = new ObservableConfigProxy(options.logger); const { appConfigs } = await loadConfig({ @@ -112,7 +103,8 @@ export async function loadBackendConfig(options: { } currentCancelFunc = resolve; - // For reloads of this module we need to use a dispose handler rather than the global. + // TODO(Rugvip): We keep this here for now to avoid breaking the old system + // since this is re-used in backend-common if (module.hot) { module.hot.addDisposeHandler(resolve); } @@ -126,11 +118,5 @@ export async function loadBackendConfig(options: { config.setConfig(ConfigReader.fromConfigs(appConfigs)); - // Subscribe to config changes and update the redaction list for logging - updateRedactionList(schema, appConfigs, options.logger); - config.subscribe(() => - updateRedactionList(schema, appConfigs, options.logger), - ); - - return config; + return { config }; } diff --git a/packages/backend-app-api/src/config/index.ts b/packages/backend-app-api/src/config/index.ts new file mode 100644 index 0000000000..0ed8d2bc93 --- /dev/null +++ b/packages/backend-app-api/src/config/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { loadBackendConfig, createConfigSecretEnumerator } from './config'; diff --git a/packages/backend-app-api/src/index.ts b/packages/backend-app-api/src/index.ts index 14e7859818..13864a3470 100644 --- a/packages/backend-app-api/src/index.ts +++ b/packages/backend-app-api/src/index.ts @@ -20,6 +20,7 @@ * @packageDocumentation */ +export * from './config'; export * from './http'; export * from './logging'; export * from './wiring'; diff --git a/packages/backend-app-api/src/logging/WinstonLogger.ts b/packages/backend-app-api/src/logging/WinstonLogger.ts index 2557ca6366..706e3cf7a7 100644 --- a/packages/backend-app-api/src/logging/WinstonLogger.ts +++ b/packages/backend-app-api/src/logging/WinstonLogger.ts @@ -68,20 +68,23 @@ export class WinstonLogger implements RootLoggerService { /** * Creates a winston log formatter for redacting secrets. */ - static redacter(): { format: Format; add: (redactions: string[]) => void } { + static redacter(): { + format: Format; + add: (redactions: Iterable) => void; + } { const redactionSet = new Set(); let redactionPattern: RegExp | undefined = undefined; return { - format: format((info: TransformableInfo) => { + format: format(info => { if (redactionPattern && typeof info.message === 'string') { info.message = info.message.replace(redactionPattern, '[REDACTED]'); } return info; })(), - add(newRedactions: string[]) { - let changed = false; + add(newRedactions) { + let added = 0; for (const redaction of newRedactions) { // Exclude secrets that are empty or just one character in length. These // typically mean that you are running local dev or tests, or using the @@ -91,18 +94,14 @@ export class WinstonLogger implements RootLoggerService { } if (!redactionSet.has(redaction)) { redactionSet.add(redaction); - changed = true; + added += 1; } } - if (changed) { - if (redactionSet.size > 0) { - redactionPattern = new RegExp( - `(${Array.from(redactionSet).join('|')})`, - 'g', - ); - } else { - redactionPattern = undefined; - } + if (added > 0) { + redactionPattern = new RegExp( + `(${Array.from(redactionSet).join('|')})`, + 'g', + ); } }, }; @@ -170,7 +169,7 @@ export class WinstonLogger implements RootLoggerService { return new WinstonLogger(this.#winston.child(meta)); } - addRedactions(redactions: string[]): void { + addRedactions(redactions: string[]) { this.#addRedactions?.(redactions); } } diff --git a/packages/backend-app-api/src/services/implementations/config/configFactory.ts b/packages/backend-app-api/src/services/implementations/config/configFactory.ts index 6bb84aa5a1..b0f5a17f3f 100644 --- a/packages/backend-app-api/src/services/implementations/config/configFactory.ts +++ b/packages/backend-app-api/src/services/implementations/config/configFactory.ts @@ -14,14 +14,28 @@ * limitations under the License. */ -import { - loadBackendConfig, - loggerToWinstonLogger, -} from '@backstage/backend-common'; import { coreServices, createServiceFactory, } from '@backstage/backend-plugin-api'; +import { LoadConfigOptionsRemote } from '@backstage/config-loader'; +import { + createConfigSecretEnumerator, + loadBackendConfig, +} from '../../../config'; + +/** @public */ +export interface ConfigFactoryOptions { + /** + * Process arguments to use instead of the default `process.argv()`. + */ + argv?: string[]; + + /** + * Enables and sets options for remote configuration loading. + */ + remote?: LoadConfigOptionsRemote; +} /** @public */ export const configFactory = createServiceFactory({ @@ -29,11 +43,19 @@ export const configFactory = createServiceFactory({ deps: { logger: coreServices.rootLogger, }, - async factory({ logger }) { - const config = await loadBackendConfig({ - argv: process.argv, - logger: loggerToWinstonLogger(logger), + async factory({ logger }, options?: ConfigFactoryOptions) { + const argv = options?.argv ?? process.argv; + const secretEnumerator = await createConfigSecretEnumerator({ logger }); + + const { config } = await loadBackendConfig({ + argv, + logger, + remote: options?.remote, }); + + logger.addRedactions(secretEnumerator(config)); + config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); + return config; }, }); diff --git a/packages/backend-app-api/src/services/implementations/config/index.ts b/packages/backend-app-api/src/services/implementations/config/index.ts index a9019b5b1c..b23c47ce0a 100644 --- a/packages/backend-app-api/src/services/implementations/config/index.ts +++ b/packages/backend-app-api/src/services/implementations/config/index.ts @@ -15,3 +15,4 @@ */ export { configFactory } from './configFactory'; +export type { ConfigFactoryOptions } from './configFactory'; diff --git a/packages/backend-plugin-api/src/services/definitions/RootLoggerService.ts b/packages/backend-plugin-api/src/services/definitions/RootLoggerService.ts index 11a12b4387..a6840f48c0 100644 --- a/packages/backend-plugin-api/src/services/definitions/RootLoggerService.ts +++ b/packages/backend-plugin-api/src/services/definitions/RootLoggerService.ts @@ -18,5 +18,5 @@ import { LoggerService } from './LoggerService'; /** @public */ export interface RootLoggerService extends LoggerService { - addRedactions(redactions: string[]): void; + addRedactions(redactions: Iterable): void; }