From e20346c25c7c08a494a1198adbf9fe075135b4dd Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Tue, 28 Feb 2023 09:22:07 -0500 Subject: [PATCH 1/9] add types to function signatures Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- packages/config-loader/src/loader.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index f5ce0eb80c..69b0e04b86 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -139,8 +139,11 @@ export async function loadConfig( const env = envFunc ?? (async (name: string) => process.env[name]); - const loadConfigFiles = async () => { - const fileConfigs = []; + const loadConfigFiles = async (): Promise<{ + fileConfigs: AppConfig[]; + loadedPaths: Set; + }> => { + const fileConfigs: AppConfig[] = []; const loadedPaths = new Set(); for (const configPath of configPaths) { @@ -176,7 +179,7 @@ export async function loadConfig( return { fileConfigs, loadedPaths }; }; - const loadRemoteConfigFiles = async () => { + const loadRemoteConfigFiles = async (): Promise => { const configs: AppConfig[] = []; const readConfigFromUrl = async (url: string) => { From 15d0bb68e5bf1332234f248af4ec1124378f6d43 Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Tue, 28 Feb 2023 09:26:48 -0500 Subject: [PATCH 2/9] allow configs to be passed directly to loadBackendConfig Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .../src/config/ObservableConfigProxy.ts | 9 ++- packages/backend-app-api/src/config/config.ts | 6 +- packages/types/src/index.ts | 1 + packages/types/src/json.ts | 58 +++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/packages/backend-app-api/src/config/ObservableConfigProxy.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.ts index dd3334c9a5..485dc60901 100644 --- a/packages/backend-app-api/src/config/ObservableConfigProxy.ts +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.ts @@ -16,7 +16,8 @@ import { ConfigService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; -import { JsonValue } from '@backstage/types'; +import type { JsonObject, JsonValue } from '@backstage/types'; +import { mergeJson } from '@backstage/types'; export class ObservableConfigProxy implements ConfigService { private config: ConfigService = new ConfigReader({}); @@ -61,6 +62,12 @@ export class ObservableConfigProxy implements ConfigService { }, }; } + mergeConfig(configData: JsonObject) { + if (this.parent) { + throw new Error('immutable'); + } + this.setConfig(new ConfigReader(mergeJson(this.config.get(), configData))); + } private select(required: true): ConfigService; private select(required: false): ConfigService | undefined; diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index d3349b41ef..50da0d78b6 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -24,10 +24,11 @@ import { ConfigTarget, LoadConfigOptionsRemote, } from '@backstage/config-loader'; -import { Config, ConfigReader } from '@backstage/config'; +import { type Config, ConfigReader } from '@backstage/config'; import { getPackages } from '@manypkg/get-packages'; import { ObservableConfigProxy } from './ObservableConfigProxy'; import { isValidUrl } from '../lib/urls'; +import type { JsonObject } from '@backstage/types'; /** @public */ export async function createConfigSecretEnumerator(options: { @@ -70,6 +71,7 @@ export async function createConfigSecretEnumerator(options: { export async function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; + config?: JsonObject; }): Promise<{ config: Config }> { const args = parseArgs(options.argv); @@ -109,12 +111,12 @@ export async function loadBackendConfig(options: { }), }, }); - console.info( `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); config.setConfig(ConfigReader.fromConfigs(appConfigs)); + config.mergeConfig(options.config ?? {}); return { config }; } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 0b5c8689b3..52b0b03622 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -21,5 +21,6 @@ */ export type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from './json'; +export { mergeJson } from './json'; export type { Observable, Observer, Subscription } from './observable'; export type { HumanDuration } from './time'; diff --git a/packages/types/src/json.ts b/packages/types/src/json.ts index bcbaaa1030..1bfe0c543f 100644 --- a/packages/types/src/json.ts +++ b/packages/types/src/json.ts @@ -41,3 +41,61 @@ export interface JsonArray extends Array {} * @public */ export type JsonValue = JsonObject | JsonArray | JsonPrimitive; + +/** + * Attempts to merge two JsonObjects together. In the case of collisions, this function + * prefers values from b, unless the value is an object, in which case it recursively + * merges the values. + * + * @param a The base object + * @param b The object to merge into a + * @returns The merged object + */ +export const mergeJson = (a: JsonObject, b: JsonObject): JsonObject => { + const final: JsonObject = {}; + const bKeys = new Set(Object.keys(b)); + const aKeys = new Set(Object.keys(a)); + const intersectingKeys = new Set([...aKeys].filter(x => bKeys.has(x))); + + // add all mutually exclusive keys to the final object + for (const key of aKeys.values()) { + if (!intersectingKeys.has(key)) { + final[key] = a[key]; + continue; + } + } + for (const key of bKeys.values()) { + if (!intersectingKeys.has(key)) { + final[key] = b[key]; + continue; + } + } + + // values now are all overlapping and are either primitives, arrays, or objects. + // for all primitives and arrays, we want to assign the value from b + // for all objects, we want to recursively merge the values + for (const key of intersectingKeys.values()) { + // check if value is an array or primitive + const value = b[key]; + if (Array.isArray(value) || typeof value !== 'object') { + final[key] = value; + continue; + } + + // check if either value is undefined and default to the defined one + const aValue = a[key]; + const bValue = b[key]; + if (!aValue) { + final[key] = bValue; + continue; + } + if (!bValue) { + final[key] = aValue; + continue; + } + + // recursively merge the values + final[key] = mergeJson(aValue as JsonObject, bValue as JsonObject); + } + return final; +}; From e504c73555c770a898c8df3f7be88548ee948c87 Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Wed, 1 Mar 2023 14:02:17 -0500 Subject: [PATCH 3/9] test merging json objects Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .changeset/spotty-cougars-wink.md | 7 ++ packages/backend-app-api/api-report.md | 1 + packages/types/api-report.md | 3 + packages/types/src/json.test.ts | 94 +++++++++++++++++++++++++- packages/types/src/json.ts | 5 +- 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 .changeset/spotty-cougars-wink.md diff --git a/.changeset/spotty-cougars-wink.md b/.changeset/spotty-cougars-wink.md new file mode 100644 index 0000000000..4e8b97a164 --- /dev/null +++ b/.changeset/spotty-cougars-wink.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-app-api': minor +'@backstage/config-loader': minor +'@backstage/types': minor +--- + +Introduces the ability to merge configs and JsonObjects diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index e99135b3ec..ed84730fc0 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -178,6 +178,7 @@ export const lifecycleServiceFactory: () => ServiceFactory< export function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; + config?: JsonObject; }): Promise<{ config: Config; }>; diff --git a/packages/types/api-report.md b/packages/types/api-report.md index 4b43f5daa6..e0b34203cf 100644 --- a/packages/types/api-report.md +++ b/packages/types/api-report.md @@ -29,6 +29,9 @@ export type JsonPrimitive = number | string | boolean | null; // @public export type JsonValue = JsonObject | JsonArray | JsonPrimitive; +// @public +export const mergeJson: (a: JsonObject, b: JsonObject) => JsonObject; + // @public export type Observable = { [Symbol.observable](): Observable; diff --git a/packages/types/src/json.test.ts b/packages/types/src/json.test.ts index 7da0ef8a3e..277992f00f 100644 --- a/packages/types/src/json.test.ts +++ b/packages/types/src/json.test.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { JsonPrimitive, JsonArray, JsonObject, JsonValue } from './json'; +import { + JsonPrimitive, + JsonArray, + JsonObject, + JsonValue, + mergeJson, +} from './json'; describe('json', () => { it('JsonPrimitive', () => { @@ -79,3 +85,89 @@ describe('json', () => { expect(true).toBe(true); }); }); + +describe('jsonMerge', () => { + it('should merge two objects', () => { + const obj1 = { a: 1, b: 2, c: 3 }; + const obj2 = { b: 4, c: 5, d: 6 }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual({ a: 1, b: 4, c: 5, d: 6 }); + }); + + it('should always prefer to merge the values of the second parameter', () => { + const obj1 = { + a: 1, + b: [1, 2, 3], + c: { + z: 1, + y: 2, + x: 3, + }, + }; + const obj2 = { + a: 2, + b: [2, 4, 6], + c: { + z: 2, + y: 4, + x: 6, + }, + }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual(obj2); + }); + + it('should prefer the second argument whenever keys collide', () => { + const obj1 = { + a: 1, + b: [1, 2, 3], + c: { + z: 1, + y: 2, + x: 3, + }, + }; + const obj2 = { + a: 2, + c: { + y: 4, + }, + }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual({ + a: 2, + b: [1, 2, 3], + c: { + z: 1, + y: 4, + x: 3, + }, + }); + }); + + it('should merge recursively', () => { + const obj1 = { + backend: { + database: { + provider: 'sqlite3', + }, + }, + }; + const obj2 = { + backend: { + database: { + password: 'password123', + }, + }, + }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual({ + backend: { + database: { + provider: 'sqlite3', + password: 'password123', + }, + }, + }); + }); +}); diff --git a/packages/types/src/json.ts b/packages/types/src/json.ts index 1bfe0c543f..4b0bb958ef 100644 --- a/packages/types/src/json.ts +++ b/packages/types/src/json.ts @@ -47,9 +47,10 @@ export type JsonValue = JsonObject | JsonArray | JsonPrimitive; * prefers values from b, unless the value is an object, in which case it recursively * merges the values. * - * @param a The base object - * @param b The object to merge into a + * @param a - The base object + * @param b - The object to merge into a * @returns The merged object + * @public */ export const mergeJson = (a: JsonObject, b: JsonObject): JsonObject => { const final: JsonObject = {}; From bf828dea7f5b7e22f0eeeeba12769f04b9f9075e Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Thu, 2 Mar 2023 09:07:09 -0500 Subject: [PATCH 4/9] test merging arrays and undefined objects Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- packages/types/src/json.test.ts | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/types/src/json.test.ts b/packages/types/src/json.test.ts index 277992f00f..1210d0a773 100644 --- a/packages/types/src/json.test.ts +++ b/packages/types/src/json.test.ts @@ -170,4 +170,39 @@ describe('jsonMerge', () => { }, }); }); + + it("should overwrite the array with the second argument's array", () => { + const obj1 = { + array: ['a', 'b', 'c'], + }; + const obj2 = { + array: [1, 2, 3], + }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual({ + array: [1, 2, 3], + }); + + const merged2 = mergeJson(obj2, obj1); + expect(merged2).toEqual({ + array: ['a', 'b', 'c'], + }); + }); + + it('should take only the defined value in the case of a collision', () => { + const obj1 = { + sqlite: undefined, + }; + const obj2 = { + sqlite: 'sqlite3', + }; + const merged = mergeJson(obj1, obj2); + expect(merged).toEqual({ + sqlite: 'sqlite3', + }); + const merged2 = mergeJson(obj2, obj1); + expect(merged2).toEqual({ + sqlite: 'sqlite3', + }); + }); }); From 1fbb7e68077b754bbd9b863936e12622d3a313fa Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Thu, 2 Mar 2023 09:34:00 -0500 Subject: [PATCH 5/9] make sure undefined valeus are weeded out first Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- packages/types/src/json.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/types/src/json.ts b/packages/types/src/json.ts index 4b0bb958ef..46b5cf9499 100644 --- a/packages/types/src/json.ts +++ b/packages/types/src/json.ts @@ -76,13 +76,6 @@ export const mergeJson = (a: JsonObject, b: JsonObject): JsonObject => { // for all primitives and arrays, we want to assign the value from b // for all objects, we want to recursively merge the values for (const key of intersectingKeys.values()) { - // check if value is an array or primitive - const value = b[key]; - if (Array.isArray(value) || typeof value !== 'object') { - final[key] = value; - continue; - } - // check if either value is undefined and default to the defined one const aValue = a[key]; const bValue = b[key]; @@ -95,6 +88,13 @@ export const mergeJson = (a: JsonObject, b: JsonObject): JsonObject => { continue; } + // check if value is an array or primitive + const value = bValue; + if (Array.isArray(value) || typeof value !== 'object') { + final[key] = value; + continue; + } + // recursively merge the values final[key] = mergeJson(aValue as JsonObject, bValue as JsonObject); } From 5c7ce5858247eefef610a69c3dec80a5db48eae2 Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Wed, 8 Mar 2023 10:07:01 -0500 Subject: [PATCH 6/9] allow an additional config to be provided to loadBackendConfig during runtime Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .changeset/curly-steaks-mate.md | 7 + .changeset/spotty-cougars-wink.md | 1 - packages/backend-app-api/api-report.md | 3 +- .../src/config/ObservableConfigProxy.ts | 9 +- packages/backend-app-api/src/config/config.ts | 12 +- packages/backend-common/api-report.md | 3 + packages/backend-common/src/config.ts | 3 +- packages/types/api-report.md | 3 - packages/types/src/index.ts | 1 - packages/types/src/json.test.ts | 129 +----------------- packages/types/src/json.ts | 59 -------- 11 files changed, 24 insertions(+), 206 deletions(-) create mode 100644 .changeset/curly-steaks-mate.md diff --git a/.changeset/curly-steaks-mate.md b/.changeset/curly-steaks-mate.md new file mode 100644 index 0000000000..6f32f27a6d --- /dev/null +++ b/.changeset/curly-steaks-mate.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-app-api': minor +'@backstage/backend-common': minor +'@backstage/config-loader': minor +--- + +Allow an additionalConfig to be provided to loadBackendConfig that fetches config values during runtime. diff --git a/.changeset/spotty-cougars-wink.md b/.changeset/spotty-cougars-wink.md index 4e8b97a164..ca3effe724 100644 --- a/.changeset/spotty-cougars-wink.md +++ b/.changeset/spotty-cougars-wink.md @@ -1,7 +1,6 @@ --- '@backstage/backend-app-api': minor '@backstage/config-loader': minor -'@backstage/types': minor --- Introduces the ability to merge configs and JsonObjects diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index ed84730fc0..6ef023333b 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -5,6 +5,7 @@ ```ts /// +import type { AppConfig } from '@backstage/config'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CacheClient } from '@backstage/backend-common'; import { Config } from '@backstage/config'; @@ -178,7 +179,7 @@ export const lifecycleServiceFactory: () => ServiceFactory< export function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; - config?: JsonObject; + additionalConfig?: AppConfig; }): Promise<{ config: Config; }>; diff --git a/packages/backend-app-api/src/config/ObservableConfigProxy.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.ts index 485dc60901..8e0ad218e9 100644 --- a/packages/backend-app-api/src/config/ObservableConfigProxy.ts +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.ts @@ -16,8 +16,7 @@ import { ConfigService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; -import type { JsonObject, JsonValue } from '@backstage/types'; -import { mergeJson } from '@backstage/types'; +import type { JsonValue } from '@backstage/types'; export class ObservableConfigProxy implements ConfigService { private config: ConfigService = new ConfigReader({}); @@ -62,12 +61,6 @@ export class ObservableConfigProxy implements ConfigService { }, }; } - mergeConfig(configData: JsonObject) { - if (this.parent) { - throw new Error('immutable'); - } - this.setConfig(new ConfigReader(mergeJson(this.config.get(), configData))); - } private select(required: true): ConfigService; private select(required: false): ConfigService | undefined; diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index 50da0d78b6..668226e8b2 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -24,11 +24,11 @@ import { ConfigTarget, LoadConfigOptionsRemote, } from '@backstage/config-loader'; -import { type Config, ConfigReader } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; +import type { Config, AppConfig } from '@backstage/config'; import { getPackages } from '@manypkg/get-packages'; import { ObservableConfigProxy } from './ObservableConfigProxy'; import { isValidUrl } from '../lib/urls'; -import type { JsonObject } from '@backstage/types'; /** @public */ export async function createConfigSecretEnumerator(options: { @@ -71,7 +71,7 @@ export async function createConfigSecretEnumerator(options: { export async function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; - config?: JsonObject; + additionalConfig?: AppConfig; }): Promise<{ config: Config }> { const args = parseArgs(options.argv); @@ -115,8 +115,12 @@ export async function loadBackendConfig(options: { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); + // add the additional config if provided + if (options.additionalConfig) { + appConfigs.push(options.additionalConfig); + } + config.setConfig(ConfigReader.fromConfigs(appConfigs)); - config.mergeConfig(options.config ?? {}); return { config }; } diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index ebdd7315c9..a37b5874a3 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -7,6 +7,8 @@ /// import { AwsCredentialsManager } from '@backstage/integration-aws-node'; +import type { AppConfig } from '@backstage/config'; +import aws from 'aws-sdk'; import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BackendFeature } from '@backstage/backend-plugin-api'; @@ -532,6 +534,7 @@ export const legacyPlugin: ( export function loadBackendConfig(options: { logger: LoggerService; remote?: LoadConfigOptionsRemote; + additionalConfig?: AppConfig; argv: string[]; }): Promise; diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index dd96a6b46a..47316cfd6e 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -19,7 +19,7 @@ import { loadBackendConfig as newLoadBackendConfig, } from '@backstage/backend-app-api'; import { LoggerService } from '@backstage/backend-plugin-api'; -import { Config } from '@backstage/config'; +import type { AppConfig, Config } from '@backstage/config'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; @@ -34,6 +34,7 @@ export async function loadBackendConfig(options: { logger: LoggerService; // process.argv or any other overrides remote?: LoadConfigOptionsRemote; + additionalConfig?: AppConfig; argv: string[]; }): Promise { const secretEnumerator = await createConfigSecretEnumerator({ diff --git a/packages/types/api-report.md b/packages/types/api-report.md index e0b34203cf..4b43f5daa6 100644 --- a/packages/types/api-report.md +++ b/packages/types/api-report.md @@ -29,9 +29,6 @@ export type JsonPrimitive = number | string | boolean | null; // @public export type JsonValue = JsonObject | JsonArray | JsonPrimitive; -// @public -export const mergeJson: (a: JsonObject, b: JsonObject) => JsonObject; - // @public export type Observable = { [Symbol.observable](): Observable; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 52b0b03622..0b5c8689b3 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -21,6 +21,5 @@ */ export type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from './json'; -export { mergeJson } from './json'; export type { Observable, Observer, Subscription } from './observable'; export type { HumanDuration } from './time'; diff --git a/packages/types/src/json.test.ts b/packages/types/src/json.test.ts index 1210d0a773..7da0ef8a3e 100644 --- a/packages/types/src/json.test.ts +++ b/packages/types/src/json.test.ts @@ -14,13 +14,7 @@ * limitations under the License. */ -import { - JsonPrimitive, - JsonArray, - JsonObject, - JsonValue, - mergeJson, -} from './json'; +import { JsonPrimitive, JsonArray, JsonObject, JsonValue } from './json'; describe('json', () => { it('JsonPrimitive', () => { @@ -85,124 +79,3 @@ describe('json', () => { expect(true).toBe(true); }); }); - -describe('jsonMerge', () => { - it('should merge two objects', () => { - const obj1 = { a: 1, b: 2, c: 3 }; - const obj2 = { b: 4, c: 5, d: 6 }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual({ a: 1, b: 4, c: 5, d: 6 }); - }); - - it('should always prefer to merge the values of the second parameter', () => { - const obj1 = { - a: 1, - b: [1, 2, 3], - c: { - z: 1, - y: 2, - x: 3, - }, - }; - const obj2 = { - a: 2, - b: [2, 4, 6], - c: { - z: 2, - y: 4, - x: 6, - }, - }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual(obj2); - }); - - it('should prefer the second argument whenever keys collide', () => { - const obj1 = { - a: 1, - b: [1, 2, 3], - c: { - z: 1, - y: 2, - x: 3, - }, - }; - const obj2 = { - a: 2, - c: { - y: 4, - }, - }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual({ - a: 2, - b: [1, 2, 3], - c: { - z: 1, - y: 4, - x: 3, - }, - }); - }); - - it('should merge recursively', () => { - const obj1 = { - backend: { - database: { - provider: 'sqlite3', - }, - }, - }; - const obj2 = { - backend: { - database: { - password: 'password123', - }, - }, - }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual({ - backend: { - database: { - provider: 'sqlite3', - password: 'password123', - }, - }, - }); - }); - - it("should overwrite the array with the second argument's array", () => { - const obj1 = { - array: ['a', 'b', 'c'], - }; - const obj2 = { - array: [1, 2, 3], - }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual({ - array: [1, 2, 3], - }); - - const merged2 = mergeJson(obj2, obj1); - expect(merged2).toEqual({ - array: ['a', 'b', 'c'], - }); - }); - - it('should take only the defined value in the case of a collision', () => { - const obj1 = { - sqlite: undefined, - }; - const obj2 = { - sqlite: 'sqlite3', - }; - const merged = mergeJson(obj1, obj2); - expect(merged).toEqual({ - sqlite: 'sqlite3', - }); - const merged2 = mergeJson(obj2, obj1); - expect(merged2).toEqual({ - sqlite: 'sqlite3', - }); - }); -}); diff --git a/packages/types/src/json.ts b/packages/types/src/json.ts index 46b5cf9499..bcbaaa1030 100644 --- a/packages/types/src/json.ts +++ b/packages/types/src/json.ts @@ -41,62 +41,3 @@ export interface JsonArray extends Array {} * @public */ export type JsonValue = JsonObject | JsonArray | JsonPrimitive; - -/** - * Attempts to merge two JsonObjects together. In the case of collisions, this function - * prefers values from b, unless the value is an object, in which case it recursively - * merges the values. - * - * @param a - The base object - * @param b - The object to merge into a - * @returns The merged object - * @public - */ -export const mergeJson = (a: JsonObject, b: JsonObject): JsonObject => { - const final: JsonObject = {}; - const bKeys = new Set(Object.keys(b)); - const aKeys = new Set(Object.keys(a)); - const intersectingKeys = new Set([...aKeys].filter(x => bKeys.has(x))); - - // add all mutually exclusive keys to the final object - for (const key of aKeys.values()) { - if (!intersectingKeys.has(key)) { - final[key] = a[key]; - continue; - } - } - for (const key of bKeys.values()) { - if (!intersectingKeys.has(key)) { - final[key] = b[key]; - continue; - } - } - - // values now are all overlapping and are either primitives, arrays, or objects. - // for all primitives and arrays, we want to assign the value from b - // for all objects, we want to recursively merge the values - for (const key of intersectingKeys.values()) { - // check if either value is undefined and default to the defined one - const aValue = a[key]; - const bValue = b[key]; - if (!aValue) { - final[key] = bValue; - continue; - } - if (!bValue) { - final[key] = aValue; - continue; - } - - // check if value is an array or primitive - const value = bValue; - if (Array.isArray(value) || typeof value !== 'object') { - final[key] = value; - continue; - } - - // recursively merge the values - final[key] = mergeJson(aValue as JsonObject, bValue as JsonObject); - } - return final; -}; From 3ca229276dcd69134d204b8eee5d04d59b302cdf Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Wed, 8 Mar 2023 14:10:36 -0500 Subject: [PATCH 7/9] allow a list of AppConfigs to be supplied at runtime Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .changeset/spotty-cougars-wink.md | 6 ------ packages/backend-app-api/api-report.md | 2 +- .../backend-app-api/src/config/ObservableConfigProxy.ts | 2 +- packages/backend-app-api/src/config/config.ts | 8 +++----- packages/backend-common/api-report.md | 3 +-- packages/backend-common/src/config.ts | 2 +- packages/config-loader/src/loader.ts | 9 +++------ 7 files changed, 10 insertions(+), 22 deletions(-) delete mode 100644 .changeset/spotty-cougars-wink.md diff --git a/.changeset/spotty-cougars-wink.md b/.changeset/spotty-cougars-wink.md deleted file mode 100644 index ca3effe724..0000000000 --- a/.changeset/spotty-cougars-wink.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/backend-app-api': minor -'@backstage/config-loader': minor ---- - -Introduces the ability to merge configs and JsonObjects diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index 6ef023333b..90b76ffb64 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -179,7 +179,7 @@ export const lifecycleServiceFactory: () => ServiceFactory< export function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; - additionalConfig?: AppConfig; + additionalConfigs?: AppConfig[]; }): Promise<{ config: Config; }>; diff --git a/packages/backend-app-api/src/config/ObservableConfigProxy.ts b/packages/backend-app-api/src/config/ObservableConfigProxy.ts index 8e0ad218e9..dd3334c9a5 100644 --- a/packages/backend-app-api/src/config/ObservableConfigProxy.ts +++ b/packages/backend-app-api/src/config/ObservableConfigProxy.ts @@ -16,7 +16,7 @@ import { ConfigService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; -import type { JsonValue } from '@backstage/types'; +import { JsonValue } from '@backstage/types'; export class ObservableConfigProxy implements ConfigService { private config: ConfigService = new ConfigReader({}); diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index 668226e8b2..a2a6f8de9e 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -71,7 +71,7 @@ export async function createConfigSecretEnumerator(options: { export async function loadBackendConfig(options: { remote?: LoadConfigOptionsRemote; argv: string[]; - additionalConfig?: AppConfig; + additionalConfigs?: AppConfig[]; }): Promise<{ config: Config }> { const args = parseArgs(options.argv); @@ -115,11 +115,9 @@ export async function loadBackendConfig(options: { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); - // add the additional config if provided - if (options.additionalConfig) { - appConfigs.push(options.additionalConfig); + if (options.additionalConfigs) { + appConfigs.push(...options.additionalConfigs); } - config.setConfig(ConfigReader.fromConfigs(appConfigs)); return { config }; diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index a37b5874a3..b1dad254f9 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -6,9 +6,8 @@ /// /// +import { AppConfig } from '@backstage/config'; import { AwsCredentialsManager } from '@backstage/integration-aws-node'; -import type { AppConfig } from '@backstage/config'; -import aws from 'aws-sdk'; import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BackendFeature } from '@backstage/backend-plugin-api'; diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 47316cfd6e..cc824c47fd 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -19,7 +19,7 @@ import { loadBackendConfig as newLoadBackendConfig, } from '@backstage/backend-app-api'; import { LoggerService } from '@backstage/backend-plugin-api'; -import type { AppConfig, Config } from '@backstage/config'; +import { AppConfig, Config } from '@backstage/config'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { setRootLoggerRedactionList } from './logging/createRootLogger'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 69b0e04b86..f5ce0eb80c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -139,11 +139,8 @@ export async function loadConfig( const env = envFunc ?? (async (name: string) => process.env[name]); - const loadConfigFiles = async (): Promise<{ - fileConfigs: AppConfig[]; - loadedPaths: Set; - }> => { - const fileConfigs: AppConfig[] = []; + const loadConfigFiles = async () => { + const fileConfigs = []; const loadedPaths = new Set(); for (const configPath of configPaths) { @@ -179,7 +176,7 @@ export async function loadConfig( return { fileConfigs, loadedPaths }; }; - const loadRemoteConfigFiles = async (): Promise => { + const loadRemoteConfigFiles = async () => { const configs: AppConfig[] = []; const readConfigFromUrl = async (url: string) => { From ffbfcd1ab7ed0ea872f5e13d6b80e5176522004d Mon Sep 17 00:00:00 2001 From: Oleg S <97077423+RobotSail@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:59:10 -0500 Subject: [PATCH 8/9] merge additional configs with remote configs Signed-off-by: Oleg S <97077423+RobotSail@users.noreply.github.com> --- .changeset/curly-steaks-mate.md | 1 - packages/backend-app-api/src/config/config.ts | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.changeset/curly-steaks-mate.md b/.changeset/curly-steaks-mate.md index 6f32f27a6d..1b41633659 100644 --- a/.changeset/curly-steaks-mate.md +++ b/.changeset/curly-steaks-mate.md @@ -1,7 +1,6 @@ --- '@backstage/backend-app-api': minor '@backstage/backend-common': minor -'@backstage/config-loader': minor --- Allow an additionalConfig to be provided to loadBackendConfig that fetches config values during runtime. diff --git a/packages/backend-app-api/src/config/config.ts b/packages/backend-app-api/src/config/config.ts index a2a6f8de9e..7367767870 100644 --- a/packages/backend-app-api/src/config/config.ts +++ b/packages/backend-app-api/src/config/config.ts @@ -94,8 +94,11 @@ export async function loadBackendConfig(options: { console.info( `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, ); - - config.setConfig(ConfigReader.fromConfigs(newConfigs)); + const configsToMerge = [...newConfigs]; + if (options.additionalConfigs) { + configsToMerge.push(...options.additionalConfigs); + } + config.setConfig(ConfigReader.fromConfigs(configsToMerge)); }, stopSignal: new Promise(resolve => { if (currentCancelFunc) { @@ -115,10 +118,11 @@ export async function loadBackendConfig(options: { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); + const finalAppConfigs = [...appConfigs]; if (options.additionalConfigs) { - appConfigs.push(...options.additionalConfigs); + finalAppConfigs.push(...options.additionalConfigs); } - config.setConfig(ConfigReader.fromConfigs(appConfigs)); + config.setConfig(ConfigReader.fromConfigs(finalAppConfigs)); return { config }; } From 0d09fec45294466edef8c37e77487a2542ba4d02 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 30 Mar 2023 17:05:56 +0200 Subject: [PATCH 9/9] Update .changeset/curly-steaks-mate.md Signed-off-by: Patrik Oldsberg --- .changeset/curly-steaks-mate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/curly-steaks-mate.md b/.changeset/curly-steaks-mate.md index 1b41633659..93e53c4959 100644 --- a/.changeset/curly-steaks-mate.md +++ b/.changeset/curly-steaks-mate.md @@ -1,6 +1,6 @@ --- -'@backstage/backend-app-api': minor -'@backstage/backend-common': minor +'@backstage/backend-app-api': patch +'@backstage/backend-common': patch --- Allow an additionalConfig to be provided to loadBackendConfig that fetches config values during runtime.