Merge pull request #7156 from backstage/mob/redact-secrets-from-logs

backend: redact secret configs from logs

.changeset/slimy-frogs-allow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Any set configurations which have been tagged with a visibility 'secret', are now redacted from log lines.

packages/backend-common/package.json

@@ -36,6 +36,7 @@
     "@backstage/integration": "^0.6.7",
     "@backstage/types": "^0.1.1",
     "@google-cloud/storage": "^5.8.0",
+    "@lerna/project": "^4.0.0",
     "@octokit/rest": "^18.5.3",
     "@types/cors": "^2.8.6",
     "@types/dockerode": "^3.2.1",

packages/backend-common/src/config.ts

@@ -18,9 +18,38 @@ import { resolve as resolvePath } from 'path';
 import parseArgs from 'minimist';
 import { Logger } from 'winston';
 import { findPaths } from '@backstage/cli-common';
-import { Config, ConfigReader } from '@backstage/config';
+import {
+  loadConfigSchema,
+  loadConfig,
+  ConfigSchema,
+} from '@backstage/config-loader';
+import { AppConfig, Config, ConfigReader } from '@backstage/config';
 import { JsonValue } from '@backstage/types';
-import { loadConfig } from '@backstage/config-loader';
+
+import { setRootLoggerRedactionList } from './logging/rootLogger';
+
+// Fetch the schema and get all the secrets to pass to the rootLogger for redaction
+const updateRedactionList = (
+  schema: ConfigSchema,
+  configs: AppConfig[],
+  logger: Logger,
+) => {
+  const secretAppConfigs = schema.process(configs, { visibility: ['secret'] });
+  const secretConfig = ConfigReader.fromConfigs(secretAppConfigs);
+  const values = new Set<string>();
+  const data = secretConfig.get();
+
+  JSON.parse(
+    JSON.stringify(data),
+    (_, v) => typeof v === 'string' && values.add(v),
+  );
+
+  logger.info(
+    `${values.size} secrets found in the config which will be redacted`,
+  );
+
+  setRootLoggerRedactionList(Array.from(values));
+};
 
 export class ObservableConfigProxy implements Config {
   private config: Config = new ConfigReader({});
@@ -151,11 +180,20 @@ export async function loadBackendConfig(options: {
   const args = parseArgs(options.argv);
   const configPaths: string[] = [args.config ?? []].flat();
 
-  const config = new ObservableConfigProxy(options.logger);
-
   /* eslint-disable-next-line no-restricted-syntax */
   const paths = findPaths(__dirname);
 
+  // TODO(hhogg): This is fetching _all_ of the packages of the monorepo
+  // in order to find the secrets for redactions, however we only care about
+  // the backend ones, we need to find a way to exclude the frontend packages.
+  const { Project } = require('@lerna/project');
+  const project = new Project(paths.targetDir);
+  const packages = await project.getPackages();
+  const schema = await loadConfigSchema({
+    dependencies: packages.map((p: any) => p.name),
+  });
+
+  const config = new ObservableConfigProxy(options.logger);
   const configs = await loadConfig({
     configRoot: paths.targetRoot,
     configPaths: configPaths.map(opt => resolvePath(opt)),
@@ -187,5 +225,9 @@ export async function loadBackendConfig(options: {
 
   config.setConfig(ConfigReader.fromConfigs(configs));
 
+  // Subscribe to config changes and update the redaction list for logging
+  updateRedactionList(schema, configs, options.logger);
+  config.subscribe(() => updateRedactionList(schema, configs, options.logger));
+
   return config;
 }

packages/backend-common/src/logging/index.ts

@@ -15,5 +15,5 @@
  */
 
 export * from './formats';
-export * from './rootLogger';
+export { createRootLogger, getRootLogger, setRootLogger } from './rootLogger';
 export * from './voidLogger';

packages/backend-common/src/logging/rootLogger.test.ts

@@ -15,7 +15,12 @@
  */
 
 import * as winston from 'winston';
-import { createRootLogger, getRootLogger, setRootLogger } from './rootLogger';
+import {
+  createRootLogger,
+  getRootLogger,
+  setRootLogger,
+  setRootLoggerRedactionList,
+} from './rootLogger';
 
 describe('rootLogger', () => {
   it('can replace the default logger', () => {
@@ -30,7 +35,20 @@ describe('rootLogger', () => {
     );
   });
 
-  describe('createRootLoger', () => {
+  it('redacts given secrets', () => {
+    const logger = createRootLogger();
+    jest.spyOn(logger, 'write');
+    setRootLoggerRedactionList(['SECRET-1', 'SECRET_2', 'SECRET.3']);
+    logger.info('Logging SECRET-1 and SECRET_2 and SECRET.3');
+
+    expect(logger.write).toHaveBeenCalledWith(
+      expect.objectContaining({
+        message: 'Logging [REDACTED] and [REDACTED] and [REDACTED]',
+      }),
+    );
+  });
+
+  describe('createRootLogger', () => {
     it('creates a new logger', () => {
       const oldLogger = getRootLogger();
       const newLogger = createRootLogger();

packages/backend-common/src/logging/rootLogger.ts

@@ -18,8 +18,10 @@ import { merge } from 'lodash';
 import * as winston from 'winston';
 import { LoggerOptions } from 'winston';
 import { coloredFormat } from './formats';
+import { escapeRegExp } from '../util/escapeRegExp';
 
 let rootLogger: winston.Logger;
+let redactionRegExp: RegExp;
 
 /** @public */
 export function getRootLogger(): winston.Logger {
@@ -31,6 +33,30 @@ export function setRootLogger(newLogger: winston.Logger) {
   rootLogger = newLogger;
 }
 
+export function setRootLoggerRedactionList(redactionList: string[]) {
+  if (redactionList.length) {
+    redactionRegExp = new RegExp(
+      `(${redactionList.map(escapeRegExp).join('|')})`,
+      'g',
+    );
+  }
+}
+
+/**
+ * A winston formatting function that finds occurrences of filteredKeys
+ * and replaces them with the corresponding identifier.
+ */
+function redactLogLine(info: winston.Logform.TransformableInfo) {
+  // TODO(hhogg): The logger is created before the config is loaded,
+  // because the logger is needed in the config loader. There is a risk of
+  // a secret being logged out during the config loading stage.
+  if (redactionRegExp) {
+    info.message = info.message.replace(redactionRegExp, '[REDACTED]');
+  }
+
+  return info;
+}
+
 /** @public */
 export function createRootLogger(
   options: winston.LoggerOptions = {},
@@ -41,6 +67,7 @@ export function createRootLogger(
       {
         level: env.LOG_LEVEL || 'info',
         format: winston.format.combine(
+          winston.format(redactLogLine)(),
           env.NODE_ENV === 'production' ? winston.format.json() : coloredFormat,
         ),
         defaultMeta: {

packages/backend-common/src/util/escapeRegExp.test.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { escapeRegExp } from './escapeRegExp';
+
+describe('escapeRegExp', () => {
+  test('does not escape non-regex characters', () => {
+    expect(escapeRegExp('Backstage Backstage')).toBe('Backstage Backstage');
+  });
+
+  test('all the characters', () => {
+    expect(escapeRegExp('^$\\.*+?()[]{}|')).toBe(
+      '\\^\\$\\\\\\.\\*\\+\\?\\(\\)\\[\\]\\{\\}\\|',
+    );
+  });
+
+  test('character: ^', () => {
+    expect(escapeRegExp('^')).toBe('\\^');
+  });
+
+  test('character: $', () => {
+    expect(escapeRegExp('$')).toBe('\\$');
+  });
+
+  test('character: \\', () => {
+    expect(escapeRegExp('\\')).toBe('\\\\');
+  });
+
+  test('character: .', () => {
+    expect(escapeRegExp('.')).toBe('\\.');
+  });
+
+  test('character: *', () => {
+    expect(escapeRegExp('*')).toBe('\\*');
+  });
+
+  test('character: +', () => {
+    expect(escapeRegExp('+')).toBe('\\+');
+  });
+
+  test('character: ?', () => {
+    expect(escapeRegExp('?')).toBe('\\?');
+  });
+
+  test('character: (', () => {
+    expect(escapeRegExp('(')).toBe('\\(');
+  });
+
+  test('character: )', () => {
+    expect(escapeRegExp(')')).toBe('\\)');
+  });
+
+  test('character: [', () => {
+    expect(escapeRegExp('[')).toBe('\\[');
+  });
+
+  test('character: ]', () => {
+    expect(escapeRegExp(']')).toBe('\\]');
+  });
+
+  test('character: {', () => {
+    expect(escapeRegExp('{')).toBe('\\{');
+  });
+
+  test('character: }', () => {
+    expect(escapeRegExp('}')).toBe('\\}');
+  });
+
+  test('character: |', () => {
+    expect(escapeRegExp('|')).toBe('\\|');
+  });
+});

packages/backend-common/src/util/escapeRegExp.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Escapes a given string to be used inside a RegExp.
+ *
+ * Taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
+ */
+export const escapeRegExp = (text: string) => {
+  return text.replace(/[.*+?^${}(\)|[\]\\]/g, '\\$&');
+};