From 2b24009b80b8d5b850f875bb65139f96f0df9b00 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 17 Mar 2023 18:01:44 +0100 Subject: [PATCH] config-loader: initial observable-based refactor Signed-off-by: Patrik Oldsberg --- packages/config-loader/package.json | 4 +- .../src/sources/ConfigSources.ts | 111 +++++++++++++ .../src/sources/EnvConfigSource.test.ts | 157 ++++++++++++++++++ .../env.ts => sources/EnvConfigSource.ts} | 24 ++- .../src/sources/FileConfigSource.ts | 19 +++ .../src/sources/MutableConfigSource.ts | 40 +++++ .../src/sources/RemoteConfigSource.ts | 19 +++ .../src/sources/SimpleBehaviorSubject.ts | 61 +++++++ .../src/sources/StaticConfigSource.ts | 61 +++++++ packages/config-loader/src/sources/types.ts | 29 ++++ yarn.lock | 4 +- 11 files changed, 525 insertions(+), 4 deletions(-) create mode 100644 packages/config-loader/src/sources/ConfigSources.ts create mode 100644 packages/config-loader/src/sources/EnvConfigSource.test.ts rename packages/config-loader/src/{lib/env.ts => sources/EnvConfigSource.ts} (83%) create mode 100644 packages/config-loader/src/sources/FileConfigSource.ts create mode 100644 packages/config-loader/src/sources/MutableConfigSource.ts create mode 100644 packages/config-loader/src/sources/RemoteConfigSource.ts create mode 100644 packages/config-loader/src/sources/SimpleBehaviorSubject.ts create mode 100644 packages/config-loader/src/sources/StaticConfigSource.ts create mode 100644 packages/config-loader/src/sources/types.ts diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 0b2946910e..44c326a03a 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -44,10 +44,12 @@ "json-schema": "^0.4.0", "json-schema-merge-allof": "^0.8.1", "json-schema-traverse": "^1.0.0", + "minimist": "^1.2.8", "node-fetch": "^2.6.7", "typescript-json-schema": "^0.55.0", "yaml": "^2.0.0", - "yup": "^0.32.9" + "yup": "^0.32.9", + "zen-observable": "^0.10.0" }, "devDependencies": { "@backstage/cli": "workspace:^", diff --git a/packages/config-loader/src/sources/ConfigSources.ts b/packages/config-loader/src/sources/ConfigSources.ts new file mode 100644 index 0000000000..061fbc991e --- /dev/null +++ b/packages/config-loader/src/sources/ConfigSources.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ObservableImpl from 'zen-observable'; +import { LoadConfigOptionsRemote } from './loader'; +import parseArgs from 'minimist'; +import { ConfigSource, ConfigSourceData } from './types'; +import { RemoteConfigSource } from './RemoteConfigSource'; +import { FileConfigSource } from './FileConfigSource'; +import { EnvConfigSource } from './EnvConfigSource'; +import { Config, ConfigReader } from '@backstage/config'; + +export class ConfigSources { + static parseArgs( + argv: string[] = process.argv, + ): Array<{ type: 'url' | 'path'; target: string }> { + const args: string[] = parseArgs(argv).config ?? []; + return args.map(target => { + try { + // eslint-disable-next-line no-new + new URL(target); + return { type: 'url', target }; + } catch { + return { type: 'path', target }; + } + }); + } + + static default(options: { + logger: Logger; + argv?: string[]; + remote?: LoadConfigOptionsRemote; + env?: Record; + }): ConfigSource { + const argSources = this.parseArgs(options.argv).map(arg => { + if (arg.type === 'url') { + if (!options.remote) { + throw new Error( + `Config argument '${arg.target}' looks like a URL but remote configuration is not enabled. Enable it by passing the \`remote\` option`, + ); + } + return RemoteConfigSource.create({ ...options, url: arg.target }); + } + return FileConfigSource.create({ ...options, path: arg.target }); + }); + const envSource = EnvConfigSource.create(options); + + return this.merge([...argSources, envSource]); + } + + static merge(sources: ConfigSource[]): ConfigSource { + return { + configData$: new ObservableImpl(observer => { + const dataArr = new Array(sources.length); + let gotAll = false; + const subscriptions = sources.map((source, i) => + source.configData$.subscribe({ + next({ data }) { + dataArr[i] = data; + if (gotAll || dataArr.every(Boolean)) { + gotAll = true; + observer.next(dataArr.flat(1)); + } + }, + error(error) { + observer.error(error); + }, + }), + ); + return () => + subscriptions.forEach(subscription => subscription.unsubscribe()); + }), + }; + } + + static toConfig(source: ConfigSource): Promise { + return new Promise((resolve, reject) => { + let config: Config | undefined = undefined; + source.configData$.subscribe({ + next({ data }) { + if (config) { + config.setConfig(ConfigReader.fromConfigs(data)); + } else { + config = ConfigReader.fromConfigs(data); + resolve(config); + } + }, + error(error) { + reject(error); + }, + }); + }); + } +} + +interface LiveConfig extends Config { + close(): void; +} diff --git a/packages/config-loader/src/sources/EnvConfigSource.test.ts b/packages/config-loader/src/sources/EnvConfigSource.test.ts new file mode 100644 index 0000000000..c1fec52a7c --- /dev/null +++ b/packages/config-loader/src/sources/EnvConfigSource.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EnvConfigSource, readEnvConfig } from './EnvConfigSource'; +import ObservableImpl from 'zen-observable'; + +describe('EnvConfigSource', () => { + it('should return empty config for empty env', async () => { + const source = EnvConfigSource.create({ env: {} }); + const spy = jest.fn(); + await ObservableImpl.from(source.configData$).forEach(d => spy(d)); + expect(spy).toHaveBeenCalledTimes(1); + expect(spy).toHaveBeenCalledWith({ data: [] }); + }); +}); + +describe('readEnvConfig', () => { + it('should return empty config for empty env', () => { + expect(readEnvConfig({})).toEqual([]); + }); + + it('should return empty config for no matching keys', () => { + expect( + readEnvConfig({ + NODE_ENV: 'production', + NOPE_ENV: 'development', + APP_CONFIG: 'foo', + APP__CONFIG_derp: 'herp', + }), + ).toEqual([]); + }); + + it('should create config from env', () => { + expect( + readEnvConfig({ + NODE_ENV: 'production', + APP_CONFIG_foo: '"bar"', + APP_CONFIG_numbers_a: '1', + APP_CONFIG_numbers_b: '2', + APP_CONFIG_numbers_c: 'false', + APP_CONFIG_numbers_d: 'abc', + APP_CONFIG_numbers_e: undefined, + APP_CONFIG_very_deep_nested_config_object: '{}', + }), + ).toEqual([ + { + data: { + foo: 'bar', + numbers: { a: 1, b: 2, c: false, d: 'abc' }, + very: { deep: { nested: { config: { object: {} } } } }, + }, + context: 'env', + }, + ]); + }); + + it('should accept string values', () => { + expect( + readEnvConfig({ APP_CONFIG_foo: '"abc"', APP_CONFIG_bar: 'xyz' }), + ).toEqual([ + { + data: { + foo: 'abc', + bar: 'xyz', + }, + context: 'env', + }, + ]); + }); + + it('should accept complex objects', () => { + expect( + readEnvConfig({ + APP_CONFIG_foo: '{ "a": 123, "b": "123", "c": [] }', + APP_CONFIG_bar: '[123, "abc", {}]', + }), + ).toEqual([ + { + data: { + foo: { a: 123, b: '123', c: [] }, + bar: [123, 'abc', {}], + }, + context: 'env', + }, + ]); + }); + + it.each([ + ['APP_CONFIG__foo'], + ['APP_CONFIG_foo_'], + ['APP_CONFIG_fo_0'], + ['APP_CONFIG_fo/o'], + ['APP_CONFIG_fo o'], + ['APP_CONFIG_foo_(foo)_foo'], + ])('should reject invalid key %p', key => { + expect(() => readEnvConfig({ [key]: '0' })).toThrow( + `Invalid env config key '${key.replace('APP_CONFIG_', '')}'`, + ); + }); + + it.each([['hello'], ['"hello'], ['{'], ['}']])( + 'should fallback to string when invalid json value %p', + value => { + expect(readEnvConfig({ APP_CONFIG_foo: value })).toEqual([ + { + data: { + foo: value, + }, + context: 'env', + }, + ]); + }, + ); + + it('should not allow null as a value', () => { + expect(() => + readEnvConfig({ + APP_CONFIG_foo: 'null', + }), + ).toThrow( + "Failed to parse JSON-serialized config value for key 'foo', Error: value may not be null", + ); + }); + + it('should not allow duplicate values', () => { + expect(() => + readEnvConfig({ + APP_CONFIG_foo_bar: '1', + APP_CONFIG_foo_bar_baz: '2', + }), + ).toThrow( + "Could not nest config for key 'foo_bar_baz' under existing value 'foo_bar'", + ); + }); + + it('should not allow mixing of objects and other values', () => { + expect(() => + readEnvConfig({ + APP_CONFIG_nested_foo: '1', + APP_CONFIG_nested: '2', + }), + ).toThrow("Refusing to override existing config at key 'nested'"); + }); +}); diff --git a/packages/config-loader/src/lib/env.ts b/packages/config-loader/src/sources/EnvConfigSource.ts similarity index 83% rename from packages/config-loader/src/lib/env.ts rename to packages/config-loader/src/sources/EnvConfigSource.ts index 638b49dbc8..c59ba3fb42 100644 --- a/packages/config-loader/src/lib/env.ts +++ b/packages/config-loader/src/sources/EnvConfigSource.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2023 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,27 @@ */ import { AppConfig } from '@backstage/config'; -import { JsonObject } from '@backstage/types'; import { assertError } from '@backstage/errors'; +import { JsonObject, Observable } from '@backstage/types'; +import ObservableImpl from 'zen-observable'; +import { ConfigSource, ConfigSourceData } from './types'; + +export function createConfigSource( + observable: Observable<{ data: ConfigSourceData[] }>, +): ConfigSource { + return { configData$: observable }; +} + +export class EnvConfigSource { + static create(options: { + env?: { + [name: string]: string | undefined; + }; + }): ConfigSource { + const data = readEnvConfig(options?.env ?? process.env); + return createConfigSource(ObservableImpl.of({ data })); + } +} const ENV_PREFIX = 'APP_CONFIG_'; @@ -42,6 +61,7 @@ const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i; * APP_CONFIG_app_title='"My Title"' * * @public + * @deprecated Use {@link EnvConfigSource} instead */ export function readEnvConfig(env: { [name: string]: string | undefined; diff --git a/packages/config-loader/src/sources/FileConfigSource.ts b/packages/config-loader/src/sources/FileConfigSource.ts new file mode 100644 index 0000000000..051cf74aef --- /dev/null +++ b/packages/config-loader/src/sources/FileConfigSource.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class FileConfigSource implements ConfigSource { + static create(options: { logger: Logger }): ConfigSource {} +} diff --git a/packages/config-loader/src/sources/MutableConfigSource.ts b/packages/config-loader/src/sources/MutableConfigSource.ts new file mode 100644 index 0000000000..1d331be1bf --- /dev/null +++ b/packages/config-loader/src/sources/MutableConfigSource.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, Observable } from '@backstage/types'; +import { SimpleBehaviorSubject } from './SimpleBehaviorSubject'; +import { ConfigSource, ConfigSourceData } from './types'; + +export class MutableConfigSource implements ConfigSource { + static create(options: { data: JsonObject; context?: string }): ConfigSource { + return new MutableConfigSource( + options.data, + options.context ?? 'mutable-config', + ); + } + + private subject: SimpleBehaviorSubject; + readonly data$: Observable; + + private constructor(initialData: JsonObject, context: string) { + this.subject = new SimpleBehaviorSubject(initialData); + this.data$ = this.subject.observable.map(data => [{ context, data }]); + } + + setData(data: JsonObject) { + this.subject.next(data); + } +} diff --git a/packages/config-loader/src/sources/RemoteConfigSource.ts b/packages/config-loader/src/sources/RemoteConfigSource.ts new file mode 100644 index 0000000000..6117af65f5 --- /dev/null +++ b/packages/config-loader/src/sources/RemoteConfigSource.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class RemoteConfigSource implements ConfigSource { + static create(options: { logger: Logger }): ConfigSource {} +} diff --git a/packages/config-loader/src/sources/SimpleBehaviorSubject.ts b/packages/config-loader/src/sources/SimpleBehaviorSubject.ts new file mode 100644 index 0000000000..54c6e8c1b4 --- /dev/null +++ b/packages/config-loader/src/sources/SimpleBehaviorSubject.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ObservableImpl from 'zen-observable'; + +/** + * A simple behavior subject that doesn't implement the observable interface itself, + * and assumes that the observable is never completed. + */ +export class SimpleBehaviorSubject { + private currentValue: T; + private terminatingError: Error | undefined; + + readonly observable: ObservableImpl; + + constructor(value: T) { + this.currentValue = value; + this.terminatingError = undefined; + this.observable = new ObservableImpl(subscriber => { + if (this.terminatingError) { + subscriber.error(this.terminatingError); + return () => {}; + } + + subscriber.next(this.currentValue); + + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); + } + + private readonly subscribers = new Set< + ZenObservable.SubscriptionObserver + >(); + + next(value: T) { + this.currentValue = value; + this.subscribers.forEach(subscriber => subscriber.next(value)); + } + + error(error: Error) { + this.terminatingError = error; + this.subscribers.forEach(subscriber => subscriber.error(error)); + this.subscribers.clear(); + } +} diff --git a/packages/config-loader/src/sources/StaticConfigSource.ts b/packages/config-loader/src/sources/StaticConfigSource.ts new file mode 100644 index 0000000000..23fa7d1cfd --- /dev/null +++ b/packages/config-loader/src/sources/StaticConfigSource.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, Observable } from '@backstage/types'; +import ObservableImpl from 'zen-observable'; +import { ConfigSource, ConfigSourceData } from './types'; + +export class StaticConfigSource implements ConfigSource { + static create(options: { + data: JsonObject | Observable | Promise; + context?: string; + }): ConfigSource { + const { data, context = 'static-config' } = options; + if (!data) { + return new StaticConfigSource(ObservableImpl.of({}), context); + } + + if ('subscribe' in data && typeof data.subscribe === 'function') { + return new StaticConfigSource( + ObservableImpl.from(data as Observable), + context, + ); + } + if ('then' in data && typeof data.then === 'function') { + return new StaticConfigSource( + new ObservableImpl(subscriber => { + (data as Promise).then( + value => subscriber.next(value), + error => subscriber.error(error), + ); + return () => {}; + }), + context, + ); + } + + return new StaticConfigSource( + ObservableImpl.of(data as JsonObject), + context, + ); + } + + readonly data$: Observable; + + private constructor(observable: ObservableImpl, context: string) { + this.data$ = observable.map(data => [{ context, data }]); + } +} diff --git a/packages/config-loader/src/sources/types.ts b/packages/config-loader/src/sources/types.ts new file mode 100644 index 0000000000..20cc04392e --- /dev/null +++ b/packages/config-loader/src/sources/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { Observable } from '@backstage/types'; + +export interface ConfigSourceData extends AppConfig { + /** + * The file path that this configuration was loaded from, if it was loaded from a file. + */ + path?: string; +} + +export interface ConfigSource { + configData$: Observable<{ data: ConfigSourceData[] }>; +} diff --git a/yarn.lock b/yarn.lock index 95a9a07c70..7059fe1a10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3939,12 +3939,14 @@ __metadata: json-schema: ^0.4.0 json-schema-merge-allof: ^0.8.1 json-schema-traverse: ^1.0.0 + minimist: ^1.2.8 mock-fs: ^5.1.0 msw: ^1.0.0 node-fetch: ^2.6.7 typescript-json-schema: ^0.55.0 yaml: ^2.0.0 yup: ^0.32.9 + zen-observable: ^0.10.0 languageName: unknown linkType: soft @@ -30752,7 +30754,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:>=1.2.2, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7": +"minimist@npm:>=1.2.2, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8": version: 1.2.8 resolution: "minimist@npm:1.2.8" checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0