diff --git a/.changeset/slimy-frogs-allow.md b/.changeset/slimy-frogs-allow.md new file mode 100644 index 0000000000..41cc0119ef --- /dev/null +++ b/.changeset/slimy-frogs-allow.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Any set configurations which have been tagged with a visibility 'secret', are now redacted from log lines. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 18a1361c98..bee663a317 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -36,6 +36,7 @@ "@backstage/integration": "^0.6.7", "@backstage/types": "^0.1.1", "@google-cloud/storage": "^5.8.0", + "@lerna/project": "^4.0.0", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", "@types/dockerode": "^3.2.1", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index fa49d5f015..cfb655cbeb 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -18,9 +18,38 @@ import { resolve as resolvePath } from 'path'; import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; -import { Config, ConfigReader } from '@backstage/config'; +import { + loadConfigSchema, + loadConfig, + ConfigSchema, +} from '@backstage/config-loader'; +import { AppConfig, Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; -import { loadConfig } from '@backstage/config-loader'; + +import { setRootLoggerRedactionList } from './logging/rootLogger'; + +// Fetch the schema and get all the secrets to pass to the rootLogger for redaction +const updateRedactionList = ( + schema: ConfigSchema, + configs: AppConfig[], + logger: Logger, +) => { + const secretAppConfigs = schema.process(configs, { visibility: ['secret'] }); + const secretConfig = ConfigReader.fromConfigs(secretAppConfigs); + const values = new Set(); + const data = secretConfig.get(); + + JSON.parse( + JSON.stringify(data), + (_, v) => typeof v === 'string' && values.add(v), + ); + + logger.info( + `${values.size} secrets found in the config which will be redacted`, + ); + + setRootLoggerRedactionList(Array.from(values)); +}; export class ObservableConfigProxy implements Config { private config: Config = new ConfigReader({}); @@ -151,11 +180,20 @@ export async function loadBackendConfig(options: { const args = parseArgs(options.argv); const configPaths: string[] = [args.config ?? []].flat(); - const config = new ObservableConfigProxy(options.logger); - /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); + // TODO(hhogg): This is fetching _all_ of the packages of the monorepo + // in order to find the secrets for redactions, however we only care about + // the backend ones, we need to find a way to exclude the frontend packages. + const { Project } = require('@lerna/project'); + const project = new Project(paths.targetDir); + const packages = await project.getPackages(); + const schema = await loadConfigSchema({ + dependencies: packages.map((p: any) => p.name), + }); + + const config = new ObservableConfigProxy(options.logger); const configs = await loadConfig({ configRoot: paths.targetRoot, configPaths: configPaths.map(opt => resolvePath(opt)), @@ -187,5 +225,9 @@ export async function loadBackendConfig(options: { config.setConfig(ConfigReader.fromConfigs(configs)); + // Subscribe to config changes and update the redaction list for logging + updateRedactionList(schema, configs, options.logger); + config.subscribe(() => updateRedactionList(schema, configs, options.logger)); + return config; } diff --git a/packages/backend-common/src/logging/index.ts b/packages/backend-common/src/logging/index.ts index 71e9618f0c..f50114a9ae 100644 --- a/packages/backend-common/src/logging/index.ts +++ b/packages/backend-common/src/logging/index.ts @@ -15,5 +15,5 @@ */ export * from './formats'; -export * from './rootLogger'; +export { createRootLogger, getRootLogger, setRootLogger } from './rootLogger'; export * from './voidLogger'; diff --git a/packages/backend-common/src/logging/rootLogger.test.ts b/packages/backend-common/src/logging/rootLogger.test.ts index 50cf867c85..192decb653 100644 --- a/packages/backend-common/src/logging/rootLogger.test.ts +++ b/packages/backend-common/src/logging/rootLogger.test.ts @@ -15,7 +15,12 @@ */ import * as winston from 'winston'; -import { createRootLogger, getRootLogger, setRootLogger } from './rootLogger'; +import { + createRootLogger, + getRootLogger, + setRootLogger, + setRootLoggerRedactionList, +} from './rootLogger'; describe('rootLogger', () => { it('can replace the default logger', () => { @@ -30,7 +35,20 @@ describe('rootLogger', () => { ); }); - describe('createRootLoger', () => { + it('redacts given secrets', () => { + const logger = createRootLogger(); + jest.spyOn(logger, 'write'); + setRootLoggerRedactionList(['SECRET-1', 'SECRET_2', 'SECRET.3']); + logger.info('Logging SECRET-1 and SECRET_2 and SECRET.3'); + + expect(logger.write).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Logging [REDACTED] and [REDACTED] and [REDACTED]', + }), + ); + }); + + describe('createRootLogger', () => { it('creates a new logger', () => { const oldLogger = getRootLogger(); const newLogger = createRootLogger(); diff --git a/packages/backend-common/src/logging/rootLogger.ts b/packages/backend-common/src/logging/rootLogger.ts index d06037863c..b05763a42c 100644 --- a/packages/backend-common/src/logging/rootLogger.ts +++ b/packages/backend-common/src/logging/rootLogger.ts @@ -18,8 +18,10 @@ import { merge } from 'lodash'; import * as winston from 'winston'; import { LoggerOptions } from 'winston'; import { coloredFormat } from './formats'; +import { escapeRegExp } from '../util/escapeRegExp'; let rootLogger: winston.Logger; +let redactionRegExp: RegExp; /** @public */ export function getRootLogger(): winston.Logger { @@ -31,6 +33,30 @@ export function setRootLogger(newLogger: winston.Logger) { rootLogger = newLogger; } +export function setRootLoggerRedactionList(redactionList: string[]) { + if (redactionList.length) { + redactionRegExp = new RegExp( + `(${redactionList.map(escapeRegExp).join('|')})`, + 'g', + ); + } +} + +/** + * A winston formatting function that finds occurrences of filteredKeys + * and replaces them with the corresponding identifier. + */ +function redactLogLine(info: winston.Logform.TransformableInfo) { + // TODO(hhogg): The logger is created before the config is loaded, + // because the logger is needed in the config loader. There is a risk of + // a secret being logged out during the config loading stage. + if (redactionRegExp) { + info.message = info.message.replace(redactionRegExp, '[REDACTED]'); + } + + return info; +} + /** @public */ export function createRootLogger( options: winston.LoggerOptions = {}, @@ -41,6 +67,7 @@ export function createRootLogger( { level: env.LOG_LEVEL || 'info', format: winston.format.combine( + winston.format(redactLogLine)(), env.NODE_ENV === 'production' ? winston.format.json() : coloredFormat, ), defaultMeta: { diff --git a/packages/backend-common/src/util/escapeRegExp.test.ts b/packages/backend-common/src/util/escapeRegExp.test.ts new file mode 100644 index 0000000000..13c6ae4a5a --- /dev/null +++ b/packages/backend-common/src/util/escapeRegExp.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { escapeRegExp } from './escapeRegExp'; + +describe('escapeRegExp', () => { + test('does not escape non-regex characters', () => { + expect(escapeRegExp('Backstage Backstage')).toBe('Backstage Backstage'); + }); + + test('all the characters', () => { + expect(escapeRegExp('^$\\.*+?()[]{}|')).toBe( + '\\^\\$\\\\\\.\\*\\+\\?\\(\\)\\[\\]\\{\\}\\|', + ); + }); + + test('character: ^', () => { + expect(escapeRegExp('^')).toBe('\\^'); + }); + + test('character: $', () => { + expect(escapeRegExp('$')).toBe('\\$'); + }); + + test('character: \\', () => { + expect(escapeRegExp('\\')).toBe('\\\\'); + }); + + test('character: .', () => { + expect(escapeRegExp('.')).toBe('\\.'); + }); + + test('character: *', () => { + expect(escapeRegExp('*')).toBe('\\*'); + }); + + test('character: +', () => { + expect(escapeRegExp('+')).toBe('\\+'); + }); + + test('character: ?', () => { + expect(escapeRegExp('?')).toBe('\\?'); + }); + + test('character: (', () => { + expect(escapeRegExp('(')).toBe('\\('); + }); + + test('character: )', () => { + expect(escapeRegExp(')')).toBe('\\)'); + }); + + test('character: [', () => { + expect(escapeRegExp('[')).toBe('\\['); + }); + + test('character: ]', () => { + expect(escapeRegExp(']')).toBe('\\]'); + }); + + test('character: {', () => { + expect(escapeRegExp('{')).toBe('\\{'); + }); + + test('character: }', () => { + expect(escapeRegExp('}')).toBe('\\}'); + }); + + test('character: |', () => { + expect(escapeRegExp('|')).toBe('\\|'); + }); +}); diff --git a/packages/backend-common/src/util/escapeRegExp.ts b/packages/backend-common/src/util/escapeRegExp.ts new file mode 100644 index 0000000000..bc78967ebe --- /dev/null +++ b/packages/backend-common/src/util/escapeRegExp.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Escapes a given string to be used inside a RegExp. + * + * Taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions + */ +export const escapeRegExp = (text: string) => { + return text.replace(/[.*+?^${}(\)|[\]\\]/g, '\\$&'); +};