paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 53 Packages Projects Releases Wiki Activity

config-loader: initial observable-based refactor

Browse Source 
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2023-03-17 18:01:44 +01:00
parent e0506af8fc
commit 2b24009b80
11 changed files with 525 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/config-loader/package.json
+3 -1
View File
@@ -44,10 +44,12 @@
"json-schema": "^0.4.0",
"json-schema-merge-allof": "^0.8.1",
"json-schema-traverse": "^1.0.0",
"minimist": "^1.2.8",
"node-fetch": "^2.6.7",
"typescript-json-schema": "^0.55.0",
"yaml": "^2.0.0",
"yup": "^0.32.9"
"yup": "^0.32.9",
"zen-observable": "^0.10.0"
},
"devDependencies": {
"@backstage/cli": "workspace:^",
packages/config-loader/src/sources/ConfigSources.ts
+111
View File
@@ -0,0 +1,111 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import ObservableImpl from 'zen-observable';
import { LoadConfigOptionsRemote } from './loader';
import parseArgs from 'minimist';
import { ConfigSource, ConfigSourceData } from './types';
import { RemoteConfigSource } from './RemoteConfigSource';
import { FileConfigSource } from './FileConfigSource';
import { EnvConfigSource } from './EnvConfigSource';
import { Config, ConfigReader } from '@backstage/config';
export class ConfigSources {
static parseArgs(
argv: string[] = process.argv,
): Array<{ type: 'url' | 'path'; target: string }> {
const args: string[] = parseArgs(argv).config ?? [];
return args.map(target => {
try {
// eslint-disable-next-line no-new
new URL(target);
return { type: 'url', target };
} catch {
return { type: 'path', target };
}
});
}
static default(options: {
logger: Logger;
argv?: string[];
remote?: LoadConfigOptionsRemote;
env?: Record<string, string>;
}): ConfigSource {
const argSources = this.parseArgs(options.argv).map(arg => {
if (arg.type === 'url') {
if (!options.remote) {
throw new Error(
`Config argument '${arg.target}' looks like a URL but remote configuration is not enabled. Enable it by passing the \`remote\` option`,
);
}
return RemoteConfigSource.create({ ...options, url: arg.target });
}
return FileConfigSource.create({ ...options, path: arg.target });
});
const envSource = EnvConfigSource.create(options);
return this.merge([...argSources, envSource]);
}
static merge(sources: ConfigSource[]): ConfigSource {
return {
configData$: new ObservableImpl<ConfigSourceData[]>(observer => {
const dataArr = new Array<ConfigSourceData[]>(sources.length);
let gotAll = false;
const subscriptions = sources.map((source, i) =>
source.configData$.subscribe({
next({ data }) {
dataArr[i] = data;
if (gotAll || dataArr.every(Boolean)) {
gotAll = true;
observer.next(dataArr.flat(1));
}
},
error(error) {
observer.error(error);
},
}),
);
return () =>
subscriptions.forEach(subscription => subscription.unsubscribe());
}),
};
}
static toConfig(source: ConfigSource): Promise<LiveConfig> {
return new Promise((resolve, reject) => {
let config: Config | undefined = undefined;
source.configData$.subscribe({
next({ data }) {
if (config) {
config.setConfig(ConfigReader.fromConfigs(data));
} else {
config = ConfigReader.fromConfigs(data);
resolve(config);
}
},
error(error) {
reject(error);
},
});
});
}
}
interface LiveConfig extends Config {
close(): void;
}
packages/config-loader/src/sources/EnvConfigSource.test.ts
+157
View File
@@ -0,0 +1,157 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EnvConfigSource, readEnvConfig } from './EnvConfigSource';
import ObservableImpl from 'zen-observable';
describe('EnvConfigSource', () => {
it('should return empty config for empty env', async () => {
const source = EnvConfigSource.create({ env: {} });
const spy = jest.fn();
await ObservableImpl.from(source.configData$).forEach(d => spy(d));
expect(spy).toHaveBeenCalledTimes(1);
expect(spy).toHaveBeenCalledWith({ data: [] });
});
});
describe('readEnvConfig', () => {
it('should return empty config for empty env', () => {
expect(readEnvConfig({})).toEqual([]);
});
it('should return empty config for no matching keys', () => {
expect(
readEnvConfig({
NODE_ENV: 'production',
NOPE_ENV: 'development',
APP_CONFIG: 'foo',
APP__CONFIG_derp: 'herp',
}),
).toEqual([]);
});
it('should create config from env', () => {
expect(
readEnvConfig({
NODE_ENV: 'production',
APP_CONFIG_foo: '"bar"',
APP_CONFIG_numbers_a: '1',
APP_CONFIG_numbers_b: '2',
APP_CONFIG_numbers_c: 'false',
APP_CONFIG_numbers_d: 'abc',
APP_CONFIG_numbers_e: undefined,
APP_CONFIG_very_deep_nested_config_object: '{}',
}),
).toEqual([
{
data: {
foo: 'bar',
numbers: { a: 1, b: 2, c: false, d: 'abc' },
very: { deep: { nested: { config: { object: {} } } } },
},
context: 'env',
},
]);
});
it('should accept string values', () => {
expect(
readEnvConfig({ APP_CONFIG_foo: '"abc"', APP_CONFIG_bar: 'xyz' }),
).toEqual([
{
data: {
foo: 'abc',
bar: 'xyz',
},
context: 'env',
},
]);
});
it('should accept complex objects', () => {
expect(
readEnvConfig({
APP_CONFIG_foo: '{ "a": 123, "b": "123", "c": [] }',
APP_CONFIG_bar: '[123, "abc", {}]',
}),
).toEqual([
{
data: {
foo: { a: 123, b: '123', c: [] },
bar: [123, 'abc', {}],
},
context: 'env',
},
]);
});
it.each([
['APP_CONFIG__foo'],
['APP_CONFIG_foo_'],
['APP_CONFIG_fo_0'],
['APP_CONFIG_fo/o'],
['APP_CONFIG_fo o'],
['APP_CONFIG_foo_(foo)_foo'],
])('should reject invalid key %p', key => {
expect(() => readEnvConfig({ [key]: '0' })).toThrow(
`Invalid env config key '${key.replace('APP_CONFIG_', '')}'`,
);
});
it.each([['hello'], ['"hello'], ['{'], ['}']])(
'should fallback to string when invalid json value %p',
value => {
expect(readEnvConfig({ APP_CONFIG_foo: value })).toEqual([
{
data: {
foo: value,
},
context: 'env',
},
]);
},
);
it('should not allow null as a value', () => {
expect(() =>
readEnvConfig({
APP_CONFIG_foo: 'null',
}),
).toThrow(
"Failed to parse JSON-serialized config value for key 'foo', Error: value may not be null",
);
});
it('should not allow duplicate values', () => {
expect(() =>
readEnvConfig({
APP_CONFIG_foo_bar: '1',
APP_CONFIG_foo_bar_baz: '2',
}),
).toThrow(
"Could not nest config for key 'foo_bar_baz' under existing value 'foo_bar'",
);
});
it('should not allow mixing of objects and other values', () => {
expect(() =>
readEnvConfig({
APP_CONFIG_nested_foo: '1',
APP_CONFIG_nested: '2',
}),
).toThrow("Refusing to override existing config at key 'nested'");
});
});
packages/config-loader/src/lib/env.ts → packages/config-loader/src/sources/EnvConfigSource.ts
+22 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 The Backstage Authors
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,8 +15,27 @@
*/
import { AppConfig } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { assertError } from '@backstage/errors';
import { JsonObject, Observable } from '@backstage/types';
import ObservableImpl from 'zen-observable';
import { ConfigSource, ConfigSourceData } from './types';
export function createConfigSource(
observable: Observable<{ data: ConfigSourceData[] }>,
): ConfigSource {
return { configData$: observable };
}
export class EnvConfigSource {
static create(options: {
env?: {
[name: string]: string | undefined;
};
}): ConfigSource {
const data = readEnvConfig(options?.env ?? process.env);
return createConfigSource(ObservableImpl.of({ data }));
}
}
const ENV_PREFIX = 'APP_CONFIG_';
@@ -42,6 +61,7 @@ const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
* APP_CONFIG_app_title='"My Title"'
*
* @public
* @deprecated Use {@link EnvConfigSource} instead
*/
export function readEnvConfig(env: {
[name: string]: string | undefined;
packages/config-loader/src/sources/FileConfigSource.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export class FileConfigSource implements ConfigSource {
static create(options: { logger: Logger }): ConfigSource {}
}
packages/config-loader/src/sources/MutableConfigSource.ts
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject, Observable } from '@backstage/types';
import { SimpleBehaviorSubject } from './SimpleBehaviorSubject';
import { ConfigSource, ConfigSourceData } from './types';
export class MutableConfigSource implements ConfigSource {
static create(options: { data: JsonObject; context?: string }): ConfigSource {
return new MutableConfigSource(
options.data,
options.context ?? 'mutable-config',
);
}
private subject: SimpleBehaviorSubject<JsonObject>;
readonly data$: Observable<ConfigSourceData[]>;
private constructor(initialData: JsonObject, context: string) {
this.subject = new SimpleBehaviorSubject(initialData);
this.data$ = this.subject.observable.map(data => [{ context, data }]);
}
setData(data: JsonObject) {
this.subject.next(data);
}
}
packages/config-loader/src/sources/RemoteConfigSource.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export class RemoteConfigSource implements ConfigSource {
static create(options: { logger: Logger }): ConfigSource {}
}
packages/config-loader/src/sources/SimpleBehaviorSubject.ts
+61
View File
@@ -0,0 +1,61 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import ObservableImpl from 'zen-observable';
/**
* A simple behavior subject that doesn't implement the observable interface itself,
* and assumes that the observable is never completed.
*/
export class SimpleBehaviorSubject<T> {
private currentValue: T;
private terminatingError: Error | undefined;
readonly observable: ObservableImpl<T>;
constructor(value: T) {
this.currentValue = value;
this.terminatingError = undefined;
this.observable = new ObservableImpl<T>(subscriber => {
if (this.terminatingError) {
subscriber.error(this.terminatingError);
return () => {};
}
subscriber.next(this.currentValue);
this.subscribers.add(subscriber);
return () => {
this.subscribers.delete(subscriber);
};
});
}
private readonly subscribers = new Set<
ZenObservable.SubscriptionObserver<T>
>();
next(value: T) {
this.currentValue = value;
this.subscribers.forEach(subscriber => subscriber.next(value));
}
error(error: Error) {
this.terminatingError = error;
this.subscribers.forEach(subscriber => subscriber.error(error));
this.subscribers.clear();
}
}
packages/config-loader/src/sources/StaticConfigSource.ts
+61
View File
@@ -0,0 +1,61 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject, Observable } from '@backstage/types';
import ObservableImpl from 'zen-observable';
import { ConfigSource, ConfigSourceData } from './types';
export class StaticConfigSource implements ConfigSource {
static create(options: {
data: JsonObject | Observable<JsonObject> | Promise<JsonObject>;
context?: string;
}): ConfigSource {
const { data, context = 'static-config' } = options;
if (!data) {
return new StaticConfigSource(ObservableImpl.of({}), context);
}
if ('subscribe' in data && typeof data.subscribe === 'function') {
return new StaticConfigSource(
ObservableImpl.from(data as Observable<JsonObject>),
context,
);
}
if ('then' in data && typeof data.then === 'function') {
return new StaticConfigSource(
new ObservableImpl(subscriber => {
(data as Promise<JsonObject>).then(
value => subscriber.next(value),
error => subscriber.error(error),
);
return () => {};
}),
context,
);
}
return new StaticConfigSource(
ObservableImpl.of(data as JsonObject),
context,
);
}
readonly data$: Observable<ConfigSourceData[]>;
private constructor(observable: ObservableImpl<JsonObject>, context: string) {
this.data$ = observable.map(data => [{ context, data }]);
}
}
packages/config-loader/src/sources/types.ts
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AppConfig } from '@backstage/config';
import { Observable } from '@backstage/types';
export interface ConfigSourceData extends AppConfig {
/**
* The file path that this configuration was loaded from, if it was loaded from a file.
*/
path?: string;
}
export interface ConfigSource {
configData$: Observable<{ data: ConfigSourceData[] }>;
}
yarn.lock
+3 -1
View File
@@ -3939,12 +3939,14 @@ __metadata:
json-schema: ^0.4.0
json-schema-merge-allof: ^0.8.1
json-schema-traverse: ^1.0.0
minimist: ^1.2.8
mock-fs: ^5.1.0
msw: ^1.0.0
node-fetch: ^2.6.7
typescript-json-schema: ^0.55.0
yaml: ^2.0.0
yup: ^0.32.9
zen-observable: ^0.10.0
languageName: unknown
linkType: soft
@@ -30752,7 +30754,7 @@ __metadata:
languageName: node
linkType: hard
"minimist@npm:>=1.2.2, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7":
"minimist@npm:>=1.2.2, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8":
version: 1.2.8
resolution: "minimist@npm:1.2.8"
checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0