diff --git a/.changeset/brave-chicken-thank.md b/.changeset/brave-chicken-thank.md new file mode 100644 index 0000000000..1278e38068 --- /dev/null +++ b/.changeset/brave-chicken-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Internal refactor of the logger and configuration loading implementations. diff --git a/.changeset/shaggy-buses-rescue.md b/.changeset/shaggy-buses-rescue.md new file mode 100644 index 0000000000..ed038c6c12 --- /dev/null +++ b/.changeset/shaggy-buses-rescue.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Moved over logging and configuration loading implementations from `@backstage/backend-common`. There is a now `WinstonLogger` which implements the `RootLoggerService` through Winston with accompanying utilities. For configuration the `loadBackendConfig` function has been moved over, but it now instead returns an object with a `config` property. diff --git a/.changeset/twelve-fans-own.md b/.changeset/twelve-fans-own.md new file mode 100644 index 0000000000..71139f73d7 --- /dev/null +++ b/.changeset/twelve-fans-own.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-plugin-api': patch +--- + +Updated the `RootLoggerService` to also have an `addRedactions` method. diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index 199b0863de..bcf746aa5a 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -12,13 +12,16 @@ import { CorsOptions } from 'cors'; import { ErrorRequestHandler } from 'express'; import { Express as Express_2 } from 'express'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { Format } from 'logform'; import { Handler } from 'express'; import { HelmetOptions } from 'helmet'; import * as http from 'http'; import { HttpRouterService } from '@backstage/backend-plugin-api'; import { IdentityService } from '@backstage/backend-plugin-api'; import { LifecycleService } from '@backstage/backend-plugin-api'; +import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { LoggerService } from '@backstage/backend-plugin-api'; +import { LogMeta } from '@backstage/backend-plugin-api'; import { PermissionsService } from '@backstage/backend-plugin-api'; import { PluginCacheManager } from '@backstage/backend-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -33,6 +36,7 @@ import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceFactoryOrFunction } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; import { TokenManagerService } from '@backstage/backend-plugin-api'; +import { transport } from 'winston'; import { UrlReader } from '@backstage/backend-common'; // @public (undocumented) @@ -51,6 +55,18 @@ export const cacheFactory: () => ServiceFactory; // @public (undocumented) export const configFactory: () => ServiceFactory; +// @public (undocumented) +export interface ConfigFactoryOptions { + argv?: string[]; + remote?: LoadConfigOptionsRemote; +} + +// @public (undocumented) +export function createConfigSecretEnumerator(options: { + logger: LoggerService; + dir?: string; +}): Promise<(config: Config) => Iterable>; + // @public export function createHttpServer( listener: RequestListener, @@ -147,6 +163,14 @@ export type IdentityFactoryOptions = { // @public export const lifecycleFactory: () => ServiceFactory; +// @public +export function loadBackendConfig(options: { + remote?: LoadConfigOptionsRemote; + argv: string[]; +}): Promise<{ + config: Config; +}>; + // @public (undocumented) export const loggerFactory: () => ServiceFactory; @@ -231,4 +255,38 @@ export const tokenManagerFactory: () => ServiceFactory; // @public (undocumented) export const urlReaderFactory: () => ServiceFactory; + +// @public +export class WinstonLogger implements RootLoggerService { + // (undocumented) + addRedactions(redactions: Iterable): void; + // (undocumented) + child(meta: LogMeta): LoggerService; + static colorFormat(): Format; + static create(options: WinstonLoggerOptions): WinstonLogger; + // (undocumented) + debug(message: string, meta?: LogMeta): void; + // (undocumented) + error(message: string, meta?: LogMeta): void; + // (undocumented) + info(message: string, meta?: LogMeta): void; + static redacter(): { + format: Format; + add: (redactions: Iterable) => void; + }; + // (undocumented) + warn(message: string, meta?: LogMeta): void; +} + +// @public (undocumented) +export interface WinstonLoggerOptions { + // (undocumented) + format: Format; + // (undocumented) + level: string; + // (undocumented) + meta?: LogMeta; + // (undocumented) + transports: transport[]; +} ``` diff --git a/packages/backend-app-api/package.json b/packages/backend-app-api/package.json index 01eaeaecaf..e7551a1130 100644 --- a/packages/backend-app-api/package.json +++ b/packages/backend-app-api/package.json @@ -36,10 +36,14 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", + "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-permission-node": "workspace:^", + "@backstage/types": "workspace:^", + "@manypkg/get-packages": "^1.1.3", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "compression": "^1.7.4", @@ -49,18 +53,22 @@ "fs-extra": "10.1.0", "helmet": "^6.0.0", "lodash": "^4.17.21", + "logform": "^2.3.2", "minimatch": "^5.0.0", + "minimist": "^1.2.5", "morgan": "^1.10.0", "node-forge": "^1.3.1", "selfsigned": "^2.0.0", "stoppable": "^1.1.0", - "winston": "^3.2.1" + "winston": "^3.2.1", + "winston-transport": "^4.5.0" }, "devDependencies": { "@backstage/cli": "workspace:^", "@types/compression": "^1.7.0", "@types/fs-extra": "^9.0.3", "@types/http-errors": "^2.0.0", + "@types/minimist": "^1.2.0", "@types/morgan": "^1.9.0", "@types/node-forge": "^1.3.0", "@types/stoppable": "^1.1.0", diff --git a/packages/backend-common/src/config.test.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.test.ts similarity index 91% rename from packages/backend-common/src/config.test.ts rename to packages/backend-app-api/src/config/ObservableConfigProxy.test.ts index 67db6996cd..876f5ac7d3 100644 --- a/packages/backend-common/src/config.test.ts +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.test.ts @@ -14,19 +14,12 @@ * limitations under the License. */ -import { LoggerService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; -import { ObservableConfigProxy } from './config'; +import { ObservableConfigProxy } from './ObservableConfigProxy'; describe('ObservableConfigProxy', () => { - const errLogger = { - error: (message: string) => { - throw new Error(message); - }, - } as unknown as LoggerService; - it('should notify subscribers', () => { - const config = new ObservableConfigProxy(errLogger); + const config = new ObservableConfigProxy(); const fn = jest.fn(); const sub = config.subscribe(fn); @@ -51,7 +44,7 @@ describe('ObservableConfigProxy', () => { }); it('should forward subscriptions', () => { - const config1 = new ObservableConfigProxy(errLogger); + const config1 = new ObservableConfigProxy(); const fn1 = jest.fn(); const fn2 = jest.fn(); @@ -119,7 +112,7 @@ describe('ObservableConfigProxy', () => { }); it('should make sub configs available as expected', () => { - const config = new ObservableConfigProxy(errLogger); + const config = new ObservableConfigProxy(); config.setConfig(new ConfigReader({ a: { x: 1 } })); diff --git a/packages/backend-app-api/src/config/ObservableConfigProxy.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.ts new file mode 100644 index 0000000000..dd3334c9a5 --- /dev/null +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigService } from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; +import { JsonValue } from '@backstage/types'; + +export class ObservableConfigProxy implements ConfigService { + private config: ConfigService = new ConfigReader({}); + + private readonly subscribers: (() => void)[] = []; + + constructor( + private readonly parent?: ObservableConfigProxy, + private parentKey?: string, + ) { + if (parent && !parentKey) { + throw new Error('parentKey is required if parent is set'); + } + } + + setConfig(config: ConfigService) { + if (this.parent) { + throw new Error('immutable'); + } + this.config = config; + for (const subscriber of this.subscribers) { + try { + subscriber(); + } catch (error) { + console.error(`Config subscriber threw error, ${error}`); + } + } + } + + subscribe(onChange: () => void): { unsubscribe: () => void } { + if (this.parent) { + return this.parent.subscribe(onChange); + } + + this.subscribers.push(onChange); + return { + unsubscribe: () => { + const index = this.subscribers.indexOf(onChange); + if (index >= 0) { + this.subscribers.splice(index, 1); + } + }, + }; + } + + private select(required: true): ConfigService; + private select(required: false): ConfigService | undefined; + private select(required: boolean): ConfigService | undefined { + if (this.parent && this.parentKey) { + if (required) { + return this.parent.select(true).getConfig(this.parentKey); + } + return this.parent.select(false)?.getOptionalConfig(this.parentKey); + } + + return this.config; + } + + has(key: string): boolean { + return this.select(false)?.has(key) ?? false; + } + keys(): string[] { + return this.select(false)?.keys() ?? []; + } + get(key?: string): T { + return this.select(true).get(key); + } + getOptional(key?: string): T | undefined { + return this.select(false)?.getOptional(key); + } + getConfig(key: string): ConfigService { + return new ObservableConfigProxy(this, key); + } + getOptionalConfig(key: string): ConfigService | undefined { + if (this.select(false)?.has(key)) { + return new ObservableConfigProxy(this, key); + } + return undefined; + } + getConfigArray(key: string): ConfigService[] { + return this.select(true).getConfigArray(key); + } + getOptionalConfigArray(key: string): ConfigService[] | undefined { + return this.select(false)?.getOptionalConfigArray(key); + } + getNumber(key: string): number { + return this.select(true).getNumber(key); + } + getOptionalNumber(key: string): number | undefined { + return this.select(false)?.getOptionalNumber(key); + } + getBoolean(key: string): boolean { + return this.select(true).getBoolean(key); + } + getOptionalBoolean(key: string): boolean | undefined { + return this.select(false)?.getOptionalBoolean(key); + } + getString(key: string): string { + return this.select(true).getString(key); + } + getOptionalString(key: string): string | undefined { + return this.select(false)?.getOptionalString(key); + } + getStringArray(key: string): string[] { + return this.select(true).getStringArray(key); + } + getOptionalStringArray(key: string): string[] | undefined { + return this.select(false)?.getOptionalStringArray(key); + } +} diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts new file mode 100644 index 0000000000..d3349b41ef --- /dev/null +++ b/packages/backend-app-api/src/config/config.ts @@ -0,0 +1,120 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import parseArgs from 'minimist'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { findPaths } from '@backstage/cli-common'; +import { + loadConfigSchema, + loadConfig, + ConfigTarget, + LoadConfigOptionsRemote, +} from '@backstage/config-loader'; +import { Config, ConfigReader } from '@backstage/config'; +import { getPackages } from '@manypkg/get-packages'; +import { ObservableConfigProxy } from './ObservableConfigProxy'; +import { isValidUrl } from '../lib/urls'; + +/** @public */ +export async function createConfigSecretEnumerator(options: { + logger: LoggerService; + dir?: string; +}): Promise<(config: Config) => Iterable> { + const { logger, dir = process.cwd() } = options; + const { packages } = await getPackages(dir); + const schema = await loadConfigSchema({ + dependencies: packages.map(p => p.packageJson.name), + }); + + return (config: Config) => { + const [secretsData] = schema.process( + [{ data: config.getOptional() ?? {}, context: 'schema-enumerator' }], + { + visibility: ['secret'], + ignoreSchemaErrors: true, + }, + ); + const secrets = new Set(); + JSON.parse( + JSON.stringify(secretsData), + (_, v) => typeof v === 'string' && secrets.add(v), + ); + logger.info( + `Found ${secrets.size} new secrets in config that will be redacted`, + ); + return secrets; + }; +} + +/** + * Load configuration for a Backend. + * + * This function should only be called once, during the initialization of the backend. + * + * @public + */ +export async function loadBackendConfig(options: { + remote?: LoadConfigOptionsRemote; + argv: string[]; +}): Promise<{ config: Config }> { + const args = parseArgs(options.argv); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); + + /* eslint-disable-next-line no-restricted-syntax */ + const paths = findPaths(__dirname); + + let currentCancelFunc: (() => void) | undefined = undefined; + + const config = new ObservableConfigProxy(); + const { appConfigs } = await loadConfig({ + configRoot: paths.targetRoot, + configTargets: configTargets, + remote: options.remote, + watch: { + onChange(newConfigs) { + console.info( + `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, + ); + + config.setConfig(ConfigReader.fromConfigs(newConfigs)); + }, + stopSignal: new Promise(resolve => { + if (currentCancelFunc) { + currentCancelFunc(); + } + currentCancelFunc = resolve; + + // TODO(Rugvip): We keep this here for now to avoid breaking the old system + // since this is re-used in backend-common + if (module.hot) { + module.hot.addDisposeHandler(resolve); + } + }), + }, + }); + + console.info( + `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, + ); + + config.setConfig(ConfigReader.fromConfigs(appConfigs)); + + return { config }; +} diff --git a/packages/backend-app-api/src/config/index.ts b/packages/backend-app-api/src/config/index.ts new file mode 100644 index 0000000000..0ed8d2bc93 --- /dev/null +++ b/packages/backend-app-api/src/config/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { loadBackendConfig, createConfigSecretEnumerator } from './config'; diff --git a/packages/backend-app-api/src/index.ts b/packages/backend-app-api/src/index.ts index 9527e35919..13864a3470 100644 --- a/packages/backend-app-api/src/index.ts +++ b/packages/backend-app-api/src/index.ts @@ -20,6 +20,8 @@ * @packageDocumentation */ +export * from './config'; export * from './http'; +export * from './logging'; export * from './wiring'; export * from './services/implementations'; diff --git a/packages/backend-app-api/src/lib/escapeRegExp.test.ts b/packages/backend-app-api/src/lib/escapeRegExp.test.ts new file mode 100644 index 0000000000..13c6ae4a5a --- /dev/null +++ b/packages/backend-app-api/src/lib/escapeRegExp.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { escapeRegExp } from './escapeRegExp'; + +describe('escapeRegExp', () => { + test('does not escape non-regex characters', () => { + expect(escapeRegExp('Backstage Backstage')).toBe('Backstage Backstage'); + }); + + test('all the characters', () => { + expect(escapeRegExp('^$\\.*+?()[]{}|')).toBe( + '\\^\\$\\\\\\.\\*\\+\\?\\(\\)\\[\\]\\{\\}\\|', + ); + }); + + test('character: ^', () => { + expect(escapeRegExp('^')).toBe('\\^'); + }); + + test('character: $', () => { + expect(escapeRegExp('$')).toBe('\\$'); + }); + + test('character: \\', () => { + expect(escapeRegExp('\\')).toBe('\\\\'); + }); + + test('character: .', () => { + expect(escapeRegExp('.')).toBe('\\.'); + }); + + test('character: *', () => { + expect(escapeRegExp('*')).toBe('\\*'); + }); + + test('character: +', () => { + expect(escapeRegExp('+')).toBe('\\+'); + }); + + test('character: ?', () => { + expect(escapeRegExp('?')).toBe('\\?'); + }); + + test('character: (', () => { + expect(escapeRegExp('(')).toBe('\\('); + }); + + test('character: )', () => { + expect(escapeRegExp(')')).toBe('\\)'); + }); + + test('character: [', () => { + expect(escapeRegExp('[')).toBe('\\['); + }); + + test('character: ]', () => { + expect(escapeRegExp(']')).toBe('\\]'); + }); + + test('character: {', () => { + expect(escapeRegExp('{')).toBe('\\{'); + }); + + test('character: }', () => { + expect(escapeRegExp('}')).toBe('\\}'); + }); + + test('character: |', () => { + expect(escapeRegExp('|')).toBe('\\|'); + }); +}); diff --git a/packages/backend-common/src/logging/voidLogger.ts b/packages/backend-app-api/src/lib/escapeRegExp.ts similarity index 66% rename from packages/backend-common/src/logging/voidLogger.ts rename to packages/backend-app-api/src/lib/escapeRegExp.ts index 79077f5c96..bc78967ebe 100644 --- a/packages/backend-common/src/logging/voidLogger.ts +++ b/packages/backend-app-api/src/lib/escapeRegExp.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,11 @@ * limitations under the License. */ -import * as winston from 'winston'; - /** - * A logger that just throws away all messages. + * Escapes a given string to be used inside a RegExp. * - * @public + * Taken from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */ -export function getVoidLogger(): winston.Logger { - return winston.createLogger({ - transports: [new winston.transports.Console({ silent: true })], - }); -} +export const escapeRegExp = (text: string) => { + return text.replace(/[.*+?^${}(\)|[\]\\]/g, '\\$&'); +}; diff --git a/packages/backend-app-api/src/lib/urls.test.ts b/packages/backend-app-api/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-app-api/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-app-api/src/lib/urls.ts b/packages/backend-app-api/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-app-api/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/backend-app-api/src/logging/WinstonLogger.ts b/packages/backend-app-api/src/logging/WinstonLogger.ts new file mode 100644 index 0000000000..cfee65d053 --- /dev/null +++ b/packages/backend-app-api/src/logging/WinstonLogger.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + LoggerService, + LogMeta, + RootLoggerService, +} from '@backstage/backend-plugin-api'; +import { Format, TransformableInfo } from 'logform'; +import { + Logger, + format, + createLogger, + transports, + transport as Transport, +} from 'winston'; + +/** + * @public + */ +export interface WinstonLoggerOptions { + meta?: LogMeta; + level: string; + format: Format; + transports: Transport[]; +} + +/** + * A {@link @backstage/backend-plugin-api#LoggerService} implementation based on winston. + * + * @public + */ +export class WinstonLogger implements RootLoggerService { + #winston: Logger; + #addRedactions?: (redactions: Iterable) => void; + + /** + * Creates a {@link WinstonLogger} instance. + */ + static create(options: WinstonLoggerOptions): WinstonLogger { + const redacter = WinstonLogger.redacter(); + + let logger = createLogger({ + level: options.level, + format: format.combine(redacter.format, options.format), + transports: options.transports ?? new transports.Console(), + }); + if (options.meta) { + logger = logger.child(options.meta); + } + + return new WinstonLogger(logger, redacter.add); + } + + /** + * Creates a winston log formatter for redacting secrets. + */ + static redacter(): { + format: Format; + add: (redactions: Iterable) => void; + } { + const redactionSet = new Set(); + + let redactionPattern: RegExp | undefined = undefined; + + return { + format: format(info => { + if (redactionPattern && typeof info.message === 'string') { + info.message = info.message.replace(redactionPattern, '[REDACTED]'); + } + return info; + })(), + add(newRedactions) { + let added = 0; + for (const redaction of newRedactions) { + // Exclude secrets that are empty or just one character in length. These + // typically mean that you are running local dev or tests, or using the + // --lax flag which sets things to just 'x'. + if (redaction.length <= 1) { + continue; + } + if (!redactionSet.has(redaction)) { + redactionSet.add(redaction); + added += 1; + } + } + if (added > 0) { + redactionPattern = new RegExp( + `(${Array.from(redactionSet).join('|')})`, + 'g', + ); + } + }, + }; + } + + /** + * Creates a pretty printed winston log formatter. + */ + static colorFormat(): Format { + const colorizer = format.colorize(); + + return format.combine( + format.timestamp(), + format.colorize({ + colors: { + timestamp: 'dim', + prefix: 'blue', + field: 'cyan', + debug: 'grey', + }, + }), + format.printf((info: TransformableInfo) => { + const { timestamp, level, message, plugin, service, ...fields } = info; + const prefix = plugin || service; + const timestampColor = colorizer.colorize('timestamp', timestamp); + const prefixColor = colorizer.colorize('prefix', prefix); + + const extraFields = Object.entries(fields) + .map( + ([key, value]) => + `${colorizer.colorize('field', `${key}`)}=${value}`, + ) + .join(' '); + + return `${timestampColor} ${prefixColor} ${level} ${message} ${extraFields}`; + }), + ); + } + + private constructor( + winston: Logger, + addRedactions?: (redactions: Iterable) => void, + ) { + this.#winston = winston; + this.#addRedactions = addRedactions; + } + + error(message: string, meta?: LogMeta): void { + this.#winston.error(message, meta); + } + + warn(message: string, meta?: LogMeta): void { + this.#winston.warn(message, meta); + } + + info(message: string, meta?: LogMeta): void { + this.#winston.info(message, meta); + } + + debug(message: string, meta?: LogMeta): void { + this.#winston.debug(message, meta); + } + + child(meta: LogMeta): LoggerService { + return new WinstonLogger(this.#winston.child(meta)); + } + + addRedactions(redactions: Iterable) { + this.#addRedactions?.(redactions); + } +} diff --git a/packages/backend-app-api/src/logging/index.ts b/packages/backend-app-api/src/logging/index.ts new file mode 100644 index 0000000000..14fe33f898 --- /dev/null +++ b/packages/backend-app-api/src/logging/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { WinstonLogger } from './WinstonLogger'; +export type { WinstonLoggerOptions } from './WinstonLogger'; diff --git a/packages/backend-app-api/src/services/implementations/config/configFactory.ts b/packages/backend-app-api/src/services/implementations/config/configFactory.ts index 6bb84aa5a1..0ec15c29fe 100644 --- a/packages/backend-app-api/src/services/implementations/config/configFactory.ts +++ b/packages/backend-app-api/src/services/implementations/config/configFactory.ts @@ -14,26 +14,34 @@ * limitations under the License. */ -import { - loadBackendConfig, - loggerToWinstonLogger, -} from '@backstage/backend-common'; import { coreServices, createServiceFactory, } from '@backstage/backend-plugin-api'; +import { LoadConfigOptionsRemote } from '@backstage/config-loader'; +import { loadBackendConfig } from '../../../config'; + +/** @public */ +export interface ConfigFactoryOptions { + /** + * Process arguments to use instead of the default `process.argv()`. + */ + argv?: string[]; + + /** + * Enables and sets options for remote configuration loading. + */ + remote?: LoadConfigOptionsRemote; +} /** @public */ export const configFactory = createServiceFactory({ service: coreServices.config, - deps: { - logger: coreServices.rootLogger, - }, - async factory({ logger }) { - const config = await loadBackendConfig({ - argv: process.argv, - logger: loggerToWinstonLogger(logger), - }); + deps: {}, + async factory({}, options?: ConfigFactoryOptions) { + const { argv = process.argv, remote } = options ?? {}; + + const { config } = await loadBackendConfig({ argv, remote }); return config; }, }); diff --git a/packages/backend-app-api/src/services/implementations/config/index.ts b/packages/backend-app-api/src/services/implementations/config/index.ts index a9019b5b1c..b23c47ce0a 100644 --- a/packages/backend-app-api/src/services/implementations/config/index.ts +++ b/packages/backend-app-api/src/services/implementations/config/index.ts @@ -15,3 +15,4 @@ */ export { configFactory } from './configFactory'; +export type { ConfigFactoryOptions } from './configFactory'; diff --git a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerFactory.ts b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerFactory.ts index 54d40b6a12..1c2214b98c 100644 --- a/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLogger/rootLoggerFactory.ts @@ -14,48 +14,37 @@ * limitations under the License. */ -import { createRootLogger } from '@backstage/backend-common'; import { createServiceFactory, - LoggerService, coreServices, } from '@backstage/backend-plugin-api'; -import { LogMeta } from '@backstage/backend-plugin-api'; -import { Logger as WinstonLogger } from 'winston'; - -class BackstageLogger implements LoggerService { - static fromWinston(logger: WinstonLogger): BackstageLogger { - return new BackstageLogger(logger); - } - - private constructor(private readonly winston: WinstonLogger) {} - - error(message: string, meta?: LogMeta): void { - this.winston.error(message, meta); - } - - warn(message: string, meta?: LogMeta): void { - this.winston.warn(message, meta); - } - - info(message: string, meta?: LogMeta): void { - this.winston.info(message, meta); - } - - debug(message: string, meta?: LogMeta): void { - this.winston.debug(message, meta); - } - - child(meta: LogMeta): LoggerService { - return new BackstageLogger(this.winston.child(meta)); - } -} +import { WinstonLogger } from '../../../logging'; +import { transports, format } from 'winston'; +import { createConfigSecretEnumerator } from '../../../config'; /** @public */ export const rootLoggerFactory = createServiceFactory({ service: coreServices.rootLogger, - deps: {}, - async factory() { - return BackstageLogger.fromWinston(createRootLogger()); + deps: { + config: coreServices.config, + }, + async factory({ config }) { + const logger = WinstonLogger.create({ + meta: { + service: 'backstage', + }, + level: process.env.LOG_LEVEL || 'info', + format: + process.env.NODE_ENV === 'production' + ? format.json() + : WinstonLogger.colorFormat(), + transports: [new transports.Console()], + }); + + const secretEnumerator = await createConfigSecretEnumerator({ logger }); + logger.addRedactions(secretEnumerator(config)); + config.subscribe?.(() => logger.addRedactions(secretEnumerator(config))); + + return logger; }, }); diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index a09557b22d..dd96a6b46a 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -14,166 +14,14 @@ * limitations under the License. */ -import { resolve as resolvePath } from 'path'; -import parseArgs from 'minimist'; -import { LoggerService } from '@backstage/backend-plugin-api'; -import { findPaths } from '@backstage/cli-common'; import { - loadConfigSchema, - loadConfig, - ConfigSchema, - ConfigTarget, - LoadConfigOptionsRemote, -} from '@backstage/config-loader'; -import { AppConfig, Config, ConfigReader } from '@backstage/config'; -import { JsonValue } from '@backstage/types'; -import { getPackages } from '@manypkg/get-packages'; - -import { isValidUrl } from './urls'; - -import { setRootLoggerRedactionList } from './logging/rootLogger'; - -// Fetch the schema and get all the secrets to pass to the rootLogger for redaction -const updateRedactionList = ( - schema: ConfigSchema, - configs: AppConfig[], - logger: LoggerService, -) => { - const secretAppConfigs = schema.process(configs, { - visibility: ['secret'], - ignoreSchemaErrors: true, - }); - const secretConfig = ConfigReader.fromConfigs(secretAppConfigs); - const values = new Set(); - const data = secretConfig.get(); - - JSON.parse( - JSON.stringify(data), - (_, v) => typeof v === 'string' && values.add(v), - ); - - logger.info( - `${values.size} secret${ - values.size > 1 ? 's' : '' - } found in the config which will be redacted`, - ); - - setRootLoggerRedactionList(Array.from(values)); -}; - -export class ObservableConfigProxy implements Config { - private config: Config = new ConfigReader({}); - - private readonly subscribers: (() => void)[] = []; - - constructor( - private readonly logger: LoggerService, - private readonly parent?: ObservableConfigProxy, - private parentKey?: string, - ) { - if (parent && !parentKey) { - throw new Error('parentKey is required if parent is set'); - } - } - - setConfig(config: Config) { - if (this.parent) { - throw new Error('immutable'); - } - this.config = config; - for (const subscriber of this.subscribers) { - try { - subscriber(); - } catch (error) { - this.logger.error(`Config subscriber threw error, ${error}`); - } - } - } - - subscribe(onChange: () => void): { unsubscribe: () => void } { - if (this.parent) { - return this.parent.subscribe(onChange); - } - - this.subscribers.push(onChange); - return { - unsubscribe: () => { - const index = this.subscribers.indexOf(onChange); - if (index >= 0) { - this.subscribers.splice(index, 1); - } - }, - }; - } - - private select(required: true): Config; - private select(required: false): Config | undefined; - private select(required: boolean): Config | undefined { - if (this.parent && this.parentKey) { - if (required) { - return this.parent.select(true).getConfig(this.parentKey); - } - return this.parent.select(false)?.getOptionalConfig(this.parentKey); - } - - return this.config; - } - - has(key: string): boolean { - return this.select(false)?.has(key) ?? false; - } - keys(): string[] { - return this.select(false)?.keys() ?? []; - } - get(key?: string): T { - return this.select(true).get(key); - } - getOptional(key?: string): T | undefined { - return this.select(false)?.getOptional(key); - } - getConfig(key: string): Config { - return new ObservableConfigProxy(this.logger, this, key); - } - getOptionalConfig(key: string): Config | undefined { - if (this.select(false)?.has(key)) { - return new ObservableConfigProxy(this.logger, this, key); - } - return undefined; - } - getConfigArray(key: string): Config[] { - return this.select(true).getConfigArray(key); - } - getOptionalConfigArray(key: string): Config[] | undefined { - return this.select(false)?.getOptionalConfigArray(key); - } - getNumber(key: string): number { - return this.select(true).getNumber(key); - } - getOptionalNumber(key: string): number | undefined { - return this.select(false)?.getOptionalNumber(key); - } - getBoolean(key: string): boolean { - return this.select(true).getBoolean(key); - } - getOptionalBoolean(key: string): boolean | undefined { - return this.select(false)?.getOptionalBoolean(key); - } - getString(key: string): string { - return this.select(true).getString(key); - } - getOptionalString(key: string): string | undefined { - return this.select(false)?.getOptionalString(key); - } - getStringArray(key: string): string[] { - return this.select(true).getStringArray(key); - } - getOptionalStringArray(key: string): string[] | undefined { - return this.select(false)?.getOptionalStringArray(key); - } -} - -// A global used to ensure that only a single file watcher is active at a time. -let currentCancelFunc: () => void; + createConfigSecretEnumerator, + loadBackendConfig as newLoadBackendConfig, +} from '@backstage/backend-app-api'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; +import { LoadConfigOptionsRemote } from '@backstage/config-loader'; +import { setRootLoggerRedactionList } from './logging/createRootLogger'; /** * Load configuration for a Backend. @@ -188,60 +36,14 @@ export async function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; }): Promise { - const args = parseArgs(options.argv); - - const configTargets: ConfigTarget[] = [args.config ?? []] - .flat() - .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); - - /* eslint-disable-next-line no-restricted-syntax */ - const paths = findPaths(__dirname); - - // TODO(hhogg): This is fetching _all_ of the packages of the monorepo - // in order to find the secrets for redactions, however we only care about - // the backend ones, we need to find a way to exclude the frontend packages. - const { packages } = await getPackages(paths.targetDir); - const schema = await loadConfigSchema({ - dependencies: packages.map(p => p.packageJson.name), + const secretEnumerator = await createConfigSecretEnumerator({ + logger: options.logger, }); + const { config } = await newLoadBackendConfig(options); - const config = new ObservableConfigProxy(options.logger); - const { appConfigs } = await loadConfig({ - configRoot: paths.targetRoot, - configTargets: configTargets, - remote: options.remote, - watch: { - onChange(newConfigs) { - options.logger.info( - `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, - ); - - config.setConfig(ConfigReader.fromConfigs(newConfigs)); - }, - stopSignal: new Promise(resolve => { - if (currentCancelFunc) { - currentCancelFunc(); - } - currentCancelFunc = resolve; - - // For reloads of this module we need to use a dispose handler rather than the global. - if (module.hot) { - module.hot.addDisposeHandler(resolve); - } - }), - }, - }); - - options.logger.info( - `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, - ); - - config.setConfig(ConfigReader.fromConfigs(appConfigs)); - - // Subscribe to config changes and update the redaction list for logging - updateRedactionList(schema, appConfigs, options.logger); - config.subscribe(() => - updateRedactionList(schema, appConfigs, options.logger), + setRootLoggerRedactionList(secretEnumerator(config)); + config.subscribe?.(() => + setRootLoggerRedactionList(secretEnumerator(config)), ); return config; diff --git a/packages/backend-common/src/logging/createRootLogger.ts b/packages/backend-common/src/logging/createRootLogger.ts new file mode 100644 index 0000000000..2e6fe20525 --- /dev/null +++ b/packages/backend-common/src/logging/createRootLogger.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { WinstonLogger } from '@backstage/backend-app-api'; +import { merge } from 'lodash'; +import * as winston from 'winston'; +import { LoggerOptions } from 'winston'; +import { setRootLogger } from './globalLoggers'; + +const redacter = WinstonLogger.redacter(); + +export const setRootLoggerRedactionList = redacter.add; + +/** + * A winston formatting function that finds occurrences of filteredKeys + * and replaces them with the corresponding identifier. + * + * @public + */ +export function redactWinstonLogLine( + info: winston.Logform.TransformableInfo, +): winston.Logform.TransformableInfo { + return redacter.format.transform(info) as winston.Logform.TransformableInfo; +} + +/** + * Creates a pretty printed winston log formatter. + * + * @public + */ +export const coloredFormat = WinstonLogger.colorFormat(); + +/** + * Creates a default "root" logger. This also calls {@link setRootLogger} under + * the hood. + * + * @remarks + * + * This is the logger instance that will be the foundation for all other logger + * instances passed to plugins etc, in a given backend. + * + * @public + */ +export function createRootLogger( + options: winston.LoggerOptions = {}, + env = process.env, +): winston.Logger { + const logger = winston + .createLogger( + merge( + { + level: env.LOG_LEVEL || 'info', + format: winston.format.combine( + redacter.format, + env.NODE_ENV === 'production' + ? winston.format.json() + : WinstonLogger.colorFormat(), + ), + transports: [ + new winston.transports.Console({ + silent: env.JEST_WORKER_ID !== undefined && !env.LOG_LEVEL, + }), + ], + }, + options, + ), + ) + .child({ service: 'backstage' }); + + setRootLogger(logger); + + return logger; +} + +setRootLogger(createRootLogger()); diff --git a/packages/backend-common/src/logging/formats.ts b/packages/backend-common/src/logging/formats.ts deleted file mode 100644 index 53eb55e790..0000000000 --- a/packages/backend-common/src/logging/formats.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as winston from 'winston'; -import { TransformableInfo } from 'logform'; - -const coloredTemplate = (info: TransformableInfo) => { - const { timestamp, level, message, plugin, service, ...fields } = info; - const colorizer = winston.format.colorize(); - const prefix = plugin || service; - const timestampColor = colorizer.colorize('timestamp', timestamp); - const prefixColor = colorizer.colorize('prefix', prefix); - - const extraFields = Object.entries(fields) - .map(([key, value]) => `${colorizer.colorize('field', `${key}`)}=${value}`) - .join(' '); - - return `${timestampColor} ${prefixColor} ${level} ${message} ${extraFields}`; -}; - -/** - * A logging format that adds coloring to console output. - * - * @public - */ -export const coloredFormat = winston.format.combine( - winston.format.timestamp(), - winston.format.colorize({ - colors: { timestamp: 'dim', prefix: 'blue', field: 'cyan', debug: 'grey' }, - }), - winston.format.printf(coloredTemplate), -); diff --git a/packages/backend-common/src/logging/globalLoggers.ts b/packages/backend-common/src/logging/globalLoggers.ts new file mode 100644 index 0000000000..7567deb646 --- /dev/null +++ b/packages/backend-common/src/logging/globalLoggers.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as winston from 'winston'; + +/** + * A logger that just throws away all messages. + * + * @public + */ +export function getVoidLogger(): winston.Logger { + return winston.createLogger({ + transports: [new winston.transports.Console({ silent: true })], + }); +} + +let rootLogger: winston.Logger; + +/** + * Gets the current root logger. + * + * @public + */ +export function getRootLogger(): winston.Logger { + return rootLogger; +} + +/** + * Sets a completely custom default "root" logger. + * + * @remarks + * + * This is the logger instance that will be the foundation for all other logger + * instances passed to plugins etc, in a given backend. + * + * Only use this if you absolutely need to make a completely custom logger. + * Normally if you want to make light adaptations to the default logger + * behavior, you would instead call {@link createRootLogger}. + * + * @public + */ +export function setRootLogger(newLogger: winston.Logger) { + rootLogger = newLogger; +} diff --git a/packages/backend-common/src/logging/index.ts b/packages/backend-common/src/logging/index.ts index ff2315c51b..e6b29afa15 100644 --- a/packages/backend-common/src/logging/index.ts +++ b/packages/backend-common/src/logging/index.ts @@ -14,12 +14,10 @@ * limitations under the License. */ -export * from './formats'; +export { getRootLogger, getVoidLogger, setRootLogger } from './globalLoggers'; export { createRootLogger, - getRootLogger, - setRootLogger, redactWinstonLogLine, -} from './rootLogger'; -export * from './voidLogger'; + coloredFormat, +} from './createRootLogger'; export { loggerToWinstonLogger } from './loggerToWinstonLogger'; diff --git a/packages/backend-common/src/logging/rootLogger.test.ts b/packages/backend-common/src/logging/rootLogger.test.ts deleted file mode 100644 index 0f2f581569..0000000000 --- a/packages/backend-common/src/logging/rootLogger.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as winston from 'winston'; -import { - createRootLogger, - getRootLogger, - setRootLogger, - setRootLoggerRedactionList, -} from './rootLogger'; - -describe('rootLogger', () => { - it('can replace the default logger', () => { - const logger = winston.createLogger(); - jest.spyOn(logger, 'info').mockReturnValue(logger); - - setRootLogger(logger); - getRootLogger().info('testing'); - - expect(logger.info).toHaveBeenCalledWith( - expect.stringContaining('testing'), - ); - }); - - it('redacts given secrets', () => { - const transport = new winston.transports.Console(); - const logger = createRootLogger({ transports: [transport] }); - jest.spyOn(transport, 'write'); - setRootLoggerRedactionList(['SECRET-1', 'SECRET_2', 'SECRET.3']); - logger.info('Logging SECRET-1 and SECRET_2 and SECRET.3'); - - expect(transport.write).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Logging [REDACTED] and [REDACTED] and [REDACTED]', - }), - ); - }); - - it('redacts but ignores empty and one-character secrets', () => { - const transport = new winston.transports.Console(); - const logger = createRootLogger({ transports: [transport] }); - jest.spyOn(transport, 'write'); - setRootLoggerRedactionList(['SECRET-1', 'SECRET_2', 'Q', '']); - logger.info('Logging SECRET-1 and SECRET_2 and Q'); - - expect(transport.write).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Logging [REDACTED] and [REDACTED] and Q', - }), - ); - }); - - describe('createRootLogger', () => { - it('creates a new logger', () => { - const oldLogger = getRootLogger(); - const newLogger = createRootLogger(); - - expect(oldLogger).not.toBe(newLogger); - }); - - it('replaces the existing root logger', () => { - const oldLogger = getRootLogger(); - createRootLogger(); - const newLogger = getRootLogger(); - expect(oldLogger).not.toBe(newLogger); - }); - - it('can append additional default metadata', () => { - const format = winston.format.json(); - const logger = createRootLogger({ - format, - defaultMeta: { - appName: 'backstage', - appEnv: 'prod', - containerId: 'abc', - }, - }); - jest.spyOn(format, 'transform'); - - logger.info('testing'); - - expect(format.transform).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'testing', - service: 'backstage', - appName: 'backstage', - appEnv: 'prod', - containerId: 'abc', - }), - {}, - ); - }); - - it('can add override existing transports', () => { - const transport = new winston.transports.Console({ level: 'debug' }); - const logger = createRootLogger({ transports: [transport] }); - expect(logger.transports.length).toBe(1); - expect(logger.transports[0]).toBe(transport); - }); - - it('can append an additional transport', () => { - const logger = createRootLogger(); - const transport = new winston.transports.Console({ level: 'debug' }); - logger.add(transport); - expect(logger.transports.length).toBe(2); - expect(logger.transports[1]).toBe(transport); - expect(logger.transports[1].level).toBe('debug'); - }); - - it('can override default format', () => { - const format = winston.format(() => false)(); - const logger = createRootLogger({ format }); - expect( - logger.format.transform({ message: 'hello', level: 'info' }), - ).toBeFalsy(); - }); - - it('can override the service label', () => { - const transport = new winston.transports.Console(); - const logger = createRootLogger({ transports: [transport] }); - const writeSpy = jest - .spyOn(transport, 'write') - .mockImplementation((_c, _e) => true); - - logger.info('msg-a'); - logger.child({ service: 'b' }).info('msg-b'); - logger.info('msg-c', { service: 'c' }); - - expect(writeSpy.mock.calls).toEqual([ - [ - expect.objectContaining({ - message: 'msg-a', - service: 'backstage', - }), - ], - [ - expect.objectContaining({ - message: 'msg-b', - service: 'b', - }), - ], - [ - expect.objectContaining({ - message: 'msg-c', - service: 'c', - }), - ], - ]); - }); - }); -}); diff --git a/packages/backend-common/src/logging/rootLogger.ts b/packages/backend-common/src/logging/rootLogger.ts deleted file mode 100644 index 6f6eff6289..0000000000 --- a/packages/backend-common/src/logging/rootLogger.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { merge } from 'lodash'; -import * as winston from 'winston'; -import { LoggerOptions } from 'winston'; -import { coloredFormat } from './formats'; -import { escapeRegExp } from '../util/escapeRegExp'; - -let rootLogger: winston.Logger; -let redactionRegExp: RegExp | undefined; - -/** - * Gets the current root logger. - * - * @public - */ -export function getRootLogger(): winston.Logger { - return rootLogger; -} - -/** - * Sets a completely custom default "root" logger. - * - * @remarks - * - * This is the logger instance that will be the foundation for all other logger - * instances passed to plugins etc, in a given backend. - * - * Only use this if you absolutely need to make a completely custom logger. - * Normally if you want to make light adaptations to the default logger - * behavior, you would instead call {@link createRootLogger}. - * - * @public - */ -export function setRootLogger(newLogger: winston.Logger) { - rootLogger = newLogger; -} - -export function setRootLoggerRedactionList(redactionList: string[]) { - // Exclude secrets that are empty or just one character in length. These - // typically mean that you are running local dev or tests, or using the - // --lax flag which sets things to just 'x'. So exclude those. - const filtered = redactionList.filter(r => r.length > 1); - - if (filtered.length) { - redactionRegExp = new RegExp( - `(${filtered.map(escapeRegExp).join('|')})`, - 'g', - ); - } else { - redactionRegExp = undefined; - } -} - -/** - * A winston formatting function that finds occurrences of filteredKeys - * and replaces them with the corresponding identifier. - * - * @public - */ -export function redactWinstonLogLine(info: winston.Logform.TransformableInfo) { - // TODO(hhogg): The logger is created before the config is loaded, because the - // logger is needed in the config loader. There is a risk of a secret being - // logged out during the config loading stage. - // TODO(freben): Added a check that info.message actually was a string, - // because it turned out that this was not necessarily guaranteed. - // https://github.com/backstage/backstage/issues/8306 - if (redactionRegExp && typeof info.message === 'string') { - info.message = info.message.replace(redactionRegExp, '[REDACTED]'); - } - - return info; -} - -/** - * Creates a default "root" logger. This also calls {@link setRootLogger} under - * the hood. - * - * @remarks - * - * This is the logger instance that will be the foundation for all other logger - * instances passed to plugins etc, in a given backend. - * - * @public - */ -export function createRootLogger( - options: winston.LoggerOptions = {}, - env = process.env, -): winston.Logger { - const logger = winston - .createLogger( - merge( - { - level: env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format(redactWinstonLogLine)(), - env.NODE_ENV === 'production' - ? winston.format.json() - : coloredFormat, - ), - transports: [ - new winston.transports.Console({ - silent: env.JEST_WORKER_ID !== undefined && !env.LOG_LEVEL, - }), - ], - }, - options, - ), - ) - .child({ service: 'backstage' }); - - setRootLogger(logger); - - return logger; -} - -rootLogger = createRootLogger(); diff --git a/packages/backend-common/src/tokens/ServerTokenManager.test.ts b/packages/backend-common/src/tokens/ServerTokenManager.test.ts index 3e5f0ba69a..3c487e9b9b 100644 --- a/packages/backend-common/src/tokens/ServerTokenManager.test.ts +++ b/packages/backend-common/src/tokens/ServerTokenManager.test.ts @@ -16,7 +16,7 @@ import { ConfigReader } from '@backstage/config'; import * as jose from 'jose'; -import { getVoidLogger } from '../logging/voidLogger'; +import { getVoidLogger } from '../logging'; import { ServerTokenManager } from './ServerTokenManager'; import { TokenManager } from './types'; diff --git a/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts b/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts new file mode 100644 index 0000000000..f5421e273b --- /dev/null +++ b/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createServiceFactory, + LoggerService, + LogMeta, + RootLoggerService, +} from '@backstage/backend-plugin-api'; + +interface MockLoggerOptions { + levels: + | boolean + | { error: boolean; warn: boolean; info: boolean; debug: boolean }; +} + +class MockLogger implements RootLoggerService { + #levels: Exclude; + #meta: LogMeta; + + error(message: string, meta?: LogMeta | Error | undefined): void { + this.#log('error', message, meta); + } + + warn(message: string, meta?: LogMeta | Error | undefined): void { + this.#log('warn', message, meta); + } + + info(message: string, meta?: LogMeta | Error | undefined): void { + this.#log('info', message, meta); + } + + debug(message: string, meta?: LogMeta | Error | undefined): void { + this.#log('debug', message, meta); + } + + child(meta: LogMeta): LoggerService { + return new MockLogger(this.#levels, { ...this.#meta, ...meta }); + } + + constructor(levels: MockLoggerOptions['levels'], meta: LogMeta) { + if (typeof levels === 'boolean') { + this.#levels = { + error: levels, + debug: levels, + info: levels, + warn: levels, + }; + } else { + this.#levels = levels; + } + this.#meta = meta; + } + + #log( + level: 'error' | 'warn' | 'info' | 'debug', + message: string, + meta?: LogMeta | Error | undefined, + ) { + if (this.#levels[level]) { + const labels = Object.entries(this.#meta) + .map(([key, value]) => `${key}=${value}`) + .join(','); + console[level](`${labels} ${message}`, meta); + } + } +} + +/** @public */ +export const mockRootLoggerService = createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + async factory(_deps) { + return new MockLogger(false, {}); + }, +}); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 8fe489750a..21650125d3 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -20,7 +20,6 @@ import { lifecycleFactory, rootLifecycleFactory, loggerFactory, - rootLoggerFactory, cacheFactory, permissionsFactory, schedulerFactory, @@ -43,6 +42,7 @@ import { } from '@backstage/backend-plugin-api'; import { mockConfigFactory } from '../implementations/mockConfigService'; +import { mockRootLoggerService } from '../implementations/mockRootLoggerService'; import { mockTokenManagerFactory } from '../implementations/mockTokenManagerService'; import { ConfigReader } from '@backstage/config'; import express from 'express'; @@ -89,10 +89,10 @@ const defaultServiceFactories = [ lifecycleFactory(), loggerFactory(), mockConfigFactory(), + mockRootLoggerService(), mockTokenManagerFactory(), permissionsFactory(), rootLifecycleFactory(), - rootLoggerFactory(), schedulerFactory(), urlReaderFactory(), ]; diff --git a/plugins/app-backend/src/service/appPlugin.test.ts b/plugins/app-backend/src/service/appPlugin.test.ts index 7c57a5d685..518f56b7f9 100644 --- a/plugins/app-backend/src/service/appPlugin.test.ts +++ b/plugins/app-backend/src/service/appPlugin.test.ts @@ -19,12 +19,6 @@ import { resolve as resolvePath } from 'path'; import fetch from 'node-fetch'; import { startTestBackend } from '@backstage/backend-test-utils'; import { appPlugin } from './appPlugin'; -import { - databaseFactory, - httpRouterFactory, - loggerFactory, - rootLoggerFactory, -} from '@backstage/backend-app-api'; describe('appPlugin', () => { beforeEach(() => { @@ -45,12 +39,6 @@ describe('appPlugin', () => { it('boots', async () => { const { server } = await startTestBackend({ - services: [ - loggerFactory(), - rootLoggerFactory(), - databaseFactory(), - httpRouterFactory(), - ], features: [ appPlugin({ appPackageName: 'app', diff --git a/yarn.lock b/yarn.lock index 282b7b24bc..d6822cad9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3381,15 +3381,20 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/cli": "workspace:^" + "@backstage/cli-common": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-permission-node": "workspace:^" + "@backstage/types": "workspace:^" + "@manypkg/get-packages": ^1.1.3 "@types/compression": ^1.7.0 "@types/cors": ^2.8.6 "@types/express": ^4.17.6 "@types/fs-extra": ^9.0.3 "@types/http-errors": ^2.0.0 + "@types/minimist": ^1.2.0 "@types/morgan": ^1.9.0 "@types/node-forge": ^1.3.0 "@types/stoppable": ^1.1.0 @@ -3401,13 +3406,16 @@ __metadata: helmet: ^6.0.0 http-errors: ^2.0.0 lodash: ^4.17.21 + logform: ^2.3.2 minimatch: ^5.0.0 + minimist: ^1.2.5 morgan: ^1.10.0 node-forge: ^1.3.1 selfsigned: ^2.0.0 stoppable: ^1.1.0 supertest: ^6.1.3 winston: ^3.2.1 + winston-transport: ^4.5.0 languageName: unknown linkType: soft