backend-app-api: hook up logger redactions and tweak surface a bit

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/backend-app-api/src/config/ObservableConfigProxy.test.ts

@@ -16,7 +16,7 @@
 
 import { LoggerService } from '@backstage/backend-plugin-api';
 import { ConfigReader } from '@backstage/config';
-import { ObservableConfigProxy } from './config';
+import { ObservableConfigProxy } from './ObservableConfigProxy';
 
 describe('ObservableConfigProxy', () => {
   const errLogger = {

packages/backend-app-api/src/config/config.ts

@@ -21,47 +21,44 @@ import { findPaths } from '@backstage/cli-common';
 import {
   loadConfigSchema,
   loadConfig,
-  ConfigSchema,
   ConfigTarget,
   LoadConfigOptionsRemote,
 } from '@backstage/config-loader';
-import { AppConfig, Config, ConfigReader } from '@backstage/config';
+import { Config, ConfigReader } from '@backstage/config';
 import { getPackages } from '@manypkg/get-packages';
 import { ObservableConfigProxy } from './ObservableConfigProxy';
 import { isValidUrl } from '../lib/urls';
 
-import { setRootLoggerRedactionList } from './logging/rootLogger';
-
-// Fetch the schema and get all the secrets to pass to the rootLogger for redaction
-const updateRedactionList = (
-  schema: ConfigSchema,
-  configs: AppConfig[],
-  logger: LoggerService,
-) => {
-  const secretAppConfigs = schema.process(configs, {
-    visibility: ['secret'],
-    ignoreSchemaErrors: true,
+/** @public */
+export async function createConfigSecretEnumerator(options: {
+  logger: LoggerService;
+  dir?: string;
+}): Promise<(config: Config) => Iterable<string>> {
+  const { logger, dir = process.cwd() } = options;
+  const { packages } = await getPackages(dir);
+  const schema = await loadConfigSchema({
+    dependencies: packages.map(p => p.packageJson.name),
   });
-  const secretConfig = ConfigReader.fromConfigs(secretAppConfigs);
-  const values = new Set<string>();
-  const data = secretConfig.get();
 
-  JSON.parse(
-    JSON.stringify(data),
-    (_, v) => typeof v === 'string' && values.add(v),
-  );
-
-  logger.info(
-    `${values.size} secret${
-      values.size > 1 ? 's' : ''
-    } found in the config which will be redacted`,
-  );
-
-  setRootLoggerRedactionList(Array.from(values));
-};
-
-// A global used to ensure that only a single file watcher is active at a time.
-let currentCancelFunc: () => void;
+  return (config: Config) => {
+    const [secretsData] = schema.process(
+      [{ data: config.get(), context: 'schema-enumerator' }],
+      {
+        visibility: ['secret'],
+        ignoreSchemaErrors: true,
+      },
+    );
+    const secrets = new Set<string>();
+    JSON.parse(
+      JSON.stringify(secretsData),
+      (_, v) => typeof v === 'string' && secrets.add(v),
+    );
+    logger.info(
+      `Found ${secrets.size} new secrets in config that will be redacted`,
+    );
+    return secrets;
+  };
+}
 
 /**
  * Load configuration for a Backend.
@@ -75,7 +72,7 @@ export async function loadBackendConfig(options: {
   // process.argv or any other overrides
   remote?: LoadConfigOptionsRemote;
   argv: string[];
-}): Promise<Config> {
+}): Promise<{ config: Config }> {
   const args = parseArgs(options.argv);
 
   const configTargets: ConfigTarget[] = [args.config ?? []]
@@ -85,13 +82,7 @@ export async function loadBackendConfig(options: {
   /* eslint-disable-next-line no-restricted-syntax */
   const paths = findPaths(__dirname);
 
-  // TODO(hhogg): This is fetching _all_ of the packages of the monorepo
-  // in order to find the secrets for redactions, however we only care about
-  // the backend ones, we need to find a way to exclude the frontend packages.
-  const { packages } = await getPackages(paths.targetDir);
-  const schema = await loadConfigSchema({
-    dependencies: packages.map(p => p.packageJson.name),
-  });
+  let currentCancelFunc: (() => void) | undefined = undefined;
 
   const config = new ObservableConfigProxy(options.logger);
   const { appConfigs } = await loadConfig({
@@ -112,7 +103,8 @@ export async function loadBackendConfig(options: {
         }
         currentCancelFunc = resolve;
 
-        // For reloads of this module we need to use a dispose handler rather than the global.
+        // TODO(Rugvip): We keep this here for now to avoid breaking the old system
+        //               since this is re-used in backend-common
         if (module.hot) {
           module.hot.addDisposeHandler(resolve);
         }
@@ -126,11 +118,5 @@ export async function loadBackendConfig(options: {
 
   config.setConfig(ConfigReader.fromConfigs(appConfigs));
 
-  // Subscribe to config changes and update the redaction list for logging
-  updateRedactionList(schema, appConfigs, options.logger);
-  config.subscribe(() =>
-    updateRedactionList(schema, appConfigs, options.logger),
-  );
-
-  return config;
+  return { config };
 }

packages/backend-app-api/src/config/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { loadBackendConfig, createConfigSecretEnumerator } from './config';

packages/backend-app-api/src/index.ts

@@ -20,6 +20,7 @@
  * @packageDocumentation
  */
 
+export * from './config';
 export * from './http';
 export * from './logging';
 export * from './wiring';

packages/backend-app-api/src/logging/WinstonLogger.ts

@@ -68,20 +68,23 @@ export class WinstonLogger implements RootLoggerService {
   /**
    * Creates a winston log formatter for redacting secrets.
    */
-  static redacter(): { format: Format; add: (redactions: string[]) => void } {
+  static redacter(): {
+    format: Format;
+    add: (redactions: Iterable<string>) => void;
+  } {
     const redactionSet = new Set<string>();
 
     let redactionPattern: RegExp | undefined = undefined;
 
     return {
-      format: format((info: TransformableInfo) => {
+      format: format(info => {
         if (redactionPattern && typeof info.message === 'string') {
           info.message = info.message.replace(redactionPattern, '[REDACTED]');
         }
         return info;
       })(),
-      add(newRedactions: string[]) {
-        let changed = false;
+      add(newRedactions) {
+        let added = 0;
         for (const redaction of newRedactions) {
           // Exclude secrets that are empty or just one character in length. These
           // typically mean that you are running local dev or tests, or using the
@@ -91,18 +94,14 @@ export class WinstonLogger implements RootLoggerService {
           }
           if (!redactionSet.has(redaction)) {
             redactionSet.add(redaction);
-            changed = true;
+            added += 1;
           }
         }
-        if (changed) {
-          if (redactionSet.size > 0) {
-            redactionPattern = new RegExp(
-              `(${Array.from(redactionSet).join('|')})`,
-              'g',
-            );
-          } else {
-            redactionPattern = undefined;
-          }
+        if (added > 0) {
+          redactionPattern = new RegExp(
+            `(${Array.from(redactionSet).join('|')})`,
+            'g',
+          );
         }
       },
     };
@@ -170,7 +169,7 @@ export class WinstonLogger implements RootLoggerService {
     return new WinstonLogger(this.#winston.child(meta));
   }
 
-  addRedactions(redactions: string[]): void {
+  addRedactions(redactions: string[]) {
     this.#addRedactions?.(redactions);
   }
 }

packages/backend-app-api/src/services/implementations/config/configFactory.ts

@@ -14,14 +14,28 @@
  * limitations under the License.
  */
 
-import {
-  loadBackendConfig,
-  loggerToWinstonLogger,
-} from '@backstage/backend-common';
 import {
   coreServices,
   createServiceFactory,
 } from '@backstage/backend-plugin-api';
+import { LoadConfigOptionsRemote } from '@backstage/config-loader';
+import {
+  createConfigSecretEnumerator,
+  loadBackendConfig,
+} from '../../../config';
+
+/** @public */
+export interface ConfigFactoryOptions {
+  /**
+   * Process arguments to use instead of the default `process.argv()`.
+   */
+  argv?: string[];
+
+  /**
+   * Enables and sets options for remote configuration loading.
+   */
+  remote?: LoadConfigOptionsRemote;
+}
 
 /** @public */
 export const configFactory = createServiceFactory({
@@ -29,11 +43,19 @@ export const configFactory = createServiceFactory({
   deps: {
     logger: coreServices.rootLogger,
   },
-  async factory({ logger }) {
-    const config = await loadBackendConfig({
-      argv: process.argv,
-      logger: loggerToWinstonLogger(logger),
+  async factory({ logger }, options?: ConfigFactoryOptions) {
+    const argv = options?.argv ?? process.argv;
+    const secretEnumerator = await createConfigSecretEnumerator({ logger });
+
+    const { config } = await loadBackendConfig({
+      argv,
+      logger,
+      remote: options?.remote,
     });
+
+    logger.addRedactions(secretEnumerator(config));
+    config.subscribe?.(() => logger.addRedactions(secretEnumerator(config)));
+
     return config;
   },
 });

packages/backend-app-api/src/services/implementations/config/index.ts

@@ -15,3 +15,4 @@
  */
 
 export { configFactory } from './configFactory';
+export type { ConfigFactoryOptions } from './configFactory';

packages/backend-plugin-api/src/services/definitions/RootLoggerService.ts

@@ -18,5 +18,5 @@ import { LoggerService } from './LoggerService';
 
 /** @public */
 export interface RootLoggerService extends LoggerService {
-  addRedactions(redactions: string[]): void;
+  addRedactions(redactions: Iterable<string>): void;
 }