diff --git a/.changeset/great-lemons-mix.md b/.changeset/great-lemons-mix.md new file mode 100644 index 0000000000..2faeea4317 --- /dev/null +++ b/.changeset/great-lemons-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Enabled live reload of locations configured in `catalog.locations`. diff --git a/.changeset/modern-ladybugs-promise.md b/.changeset/modern-ladybugs-promise.md new file mode 100644 index 0000000000..8ad48d42f4 --- /dev/null +++ b/.changeset/modern-ladybugs-promise.md @@ -0,0 +1,5 @@ +--- +'@backstage/config': patch +--- + +Extended the `Config` interface to have an optional `subscribe` method that can be used be notified of updates to the configuration. diff --git a/.changeset/thin-bugs-compete.md b/.changeset/thin-bugs-compete.md new file mode 100644 index 0000000000..8b9805249c --- /dev/null +++ b/.changeset/thin-bugs-compete.md @@ -0,0 +1,5 @@ +--- +'@backstage/config-loader': patch +--- + +Add support for config file watching through a new group of `watch` options to `loadConfig`. diff --git a/.changeset/tricky-starfishes-cheer.md b/.changeset/tricky-starfishes-cheer.md new file mode 100644 index 0000000000..bc2fb6a91b --- /dev/null +++ b/.changeset/tricky-starfishes-cheer.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Add support for watching configuration by implementing the `subscribe` method in the configuration returned by `loadBackendConfig`. diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 1316642ae1..7c660ee4e6 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -18,32 +18,142 @@ import { resolve as resolvePath } from 'path'; import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; -import { Config, ConfigReader } from '@backstage/config'; +import { Config, ConfigReader, JsonValue } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; +class ObservableConfigProxy implements Config { + private config: Config = new ConfigReader({}); + + private readonly subscribers: (() => void)[] = []; + + constructor(private readonly logger: Logger) {} + + setConfig(config: Config) { + this.config = config; + for (const subscriber of this.subscribers) { + try { + subscriber(); + } catch (error) { + this.logger.error(`Config subscriber threw error, ${error}`); + } + } + } + + subscribe(onChange: () => void): { unsubscribe: () => void } { + this.subscribers.push(onChange); + return { + unsubscribe: () => { + const index = this.subscribers.indexOf(onChange); + if (index >= 0) { + this.subscribers.splice(index, 1); + } + }, + }; + } + + has(key: string): boolean { + return this.config.has(key); + } + keys(): string[] { + return this.config.keys(); + } + get(key?: string): T { + return this.config.get(key); + } + getOptional(key?: string): T | undefined { + return this.config.getOptional(key); + } + getConfig(key: string): Config { + return this.config.getConfig(key); + } + getOptionalConfig(key: string): Config | undefined { + return this.config.getOptionalConfig(key); + } + getConfigArray(key: string): Config[] { + return this.config.getConfigArray(key); + } + getOptionalConfigArray(key: string): Config[] | undefined { + return this.config.getOptionalConfigArray(key); + } + getNumber(key: string): number { + return this.config.getNumber(key); + } + getOptionalNumber(key: string): number | undefined { + return this.config.getOptionalNumber(key); + } + getBoolean(key: string): boolean { + return this.config.getBoolean(key); + } + getOptionalBoolean(key: string): boolean | undefined { + return this.config.getOptionalBoolean(key); + } + getString(key: string): string { + return this.config.getString(key); + } + getOptionalString(key: string): string | undefined { + return this.config.getOptionalString(key); + } + getStringArray(key: string): string[] { + return this.config.getStringArray(key); + } + getOptionalStringArray(key: string): string[] | undefined { + return this.config.getOptionalStringArray(key); + } +} + type Options = { logger: Logger; // process.argv or any other overrides argv: string[]; }; +// A global used to ensure that only a single file watcher is active at a time. +let currentCancelFunc: () => void; + /** - * Load configuration for a Backend + * Load configuration for a Backend. + * + * This function should only be called once, during the initialization of the backend. */ export async function loadBackendConfig(options: Options): Promise { const args = parseArgs(options.argv); - const configOpts: string[] = [args.config ?? []].flat(); + const configPaths: string[] = [args.config ?? []].flat(); + + const config = new ObservableConfigProxy(options.logger); /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); + const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configOpts.map(opt => resolvePath(opt)), + configPaths: configPaths.map(opt => resolvePath(opt)), + watch: { + onChange(newConfigs) { + options.logger.info( + `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, + ); + + config.setConfig(ConfigReader.fromConfigs(newConfigs)); + }, + stopSignal: new Promise(resolve => { + if (currentCancelFunc) { + currentCancelFunc(); + } + currentCancelFunc = resolve; + + // For reloads of this module we need to use a dispose handler rather than the global. + if (module.hot) { + module.hot.addDisposeHandler(resolve); + } + }), + }, }); options.logger.info( `Loaded config from ${configs.map(c => c.context).join(', ')}`, ); - return ConfigReader.fromConfigs(configs); + config.setConfig(ConfigReader.fromConfigs(configs)); + + return config; } diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 47bc22c60f..38aa0b1ebb 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -37,6 +37,10 @@ export type LoadConfigOptions = { configPaths: string[]; env?: string; experimentalEnvFunc?: EnvFunc; + watch?: { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; + }; }; // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 33b8acc3b9..d4c737d2c8 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,6 +34,7 @@ "@backstage/config": "^0.1.6", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", + "chokidar": "^3.5.2", "fs-extra": "9.1.0", "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index b0e236f9c9..3f51e76936 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ +import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; +import fs from 'fs-extra'; describe('loadConfig', () => { - beforeAll(() => { + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; @@ -63,7 +65,7 @@ describe('loadConfig', () => { }); }); - afterAll(() => { + afterEach(() => { mockFs.restore(); }); @@ -170,4 +172,81 @@ describe('loadConfig', () => { }, ]); }); + + it('watches config files', async () => { + const onChange = defer(); + const stopSignal = defer(); + + await expect( + loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await expect(onChange.promise).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'New Title', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + + it('stops watching config files', async () => { + const stopSignal = defer(); + + await loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: () => { + expect('not').toBe('called'); + }, + stopSignal: stopSignal.promise, + }, + }); + + stopSignal.resolve(); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await new Promise(resolve => setTimeout(resolve, 1000)); + }); + + function defer() { + let resolve: (value: T) => void; + const promise = new Promise(_resolve => { + resolve = _resolve; + }); + return { promise, resolve: resolve! }; + } }); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 3846d6474b..4dbfc007a7 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -16,6 +16,7 @@ import fs from 'fs-extra'; import yaml from 'yaml'; +import chokidar from 'chokidar'; import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; import { AppConfig } from '@backstage/config'; import { @@ -42,13 +43,27 @@ export type LoadConfigOptions = { * @experimental This API is not stable and may change at any point */ experimentalEnvFunc?: EnvFunc; + + /** + * An optional configuration that enables watching of config files. + */ + watch?: { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; + }; }; export async function loadConfig( options: LoadConfigOptions, ): Promise { - const configs = []; - const { configRoot, experimentalEnvFunc: envFunc } = options; + const { configRoot, experimentalEnvFunc: envFunc, watch } = options; const configPaths = options.configPaths.slice(); // If no paths are provided, we default to reading @@ -64,7 +79,9 @@ export async function loadConfig( const env = envFunc ?? (async (name: string) => process.env[name]); - try { + const loadConfigFiles = async () => { + const configs = []; + for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); @@ -83,13 +100,50 @@ export async function loadConfig( configs.push({ data, context: basename(configPath) }); } + + return configs; + }; + + let fileConfigs; + try { + fileConfigs = await loadConfigFiles(); } catch (error) { throw new Error( `Failed to read static configuration file, ${error.message}`, ); } - configs.push(...readEnvConfig(process.env)); + const envConfigs = await readEnvConfig(process.env); - return configs; + // Set up config file watching if requested by the caller + if (watch) { + let currentSerializedConfig = JSON.stringify(fileConfigs); + + const watcher = chokidar.watch(configPaths, { + usePolling: process.env.NODE_ENV === 'test', + }); + watcher.on('change', async () => { + try { + const newConfigs = await loadConfigFiles(); + const newSerializedConfig = JSON.stringify(newConfigs); + + if (currentSerializedConfig === newSerializedConfig) { + return; + } + currentSerializedConfig = newSerializedConfig; + + watch.onChange([...newConfigs, ...envConfigs]); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + }); + + if (watch.stopSignal) { + watch.stopSignal.then(() => { + watcher.close(); + }); + } + } + + return [...fileConfigs, ...envConfigs]; } diff --git a/packages/config/api-report.md b/packages/config/api-report.md index eff4236026..91f37894c0 100644 --- a/packages/config/api-report.md +++ b/packages/config/api-report.md @@ -16,6 +16,9 @@ export type AppConfig = { // // @public (undocumented) export type Config = { + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; has(key: string): boolean; keys(): string[]; get(key?: string): T; diff --git a/packages/config/src/types.ts b/packages/config/src/types.ts index 1135e7d554..84de5dae70 100644 --- a/packages/config/src/types.ts +++ b/packages/config/src/types.ts @@ -26,6 +26,17 @@ export type AppConfig = { }; export type Config = { + /** + * Subscribes to the configuration object in order to receive a notification + * whenever any value within the configuration has changed. + * + * This method is optional to implement, and consumers need to check if it is + * implemented before invoking it. + */ + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; + has(key: string): boolean; keys(): string[]; diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts index 32ce28dc1b..ef3e47eb8b 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts @@ -70,4 +70,61 @@ describe('ConfigLocationEntityProvider', () => { ]), }); }); + + it('should be able to observe the config', async () => { + // Grab the subscriber function and use mutable config data to mock a config file change + let subscriber: () => void; + const mutableConfigData = { + catalog: { + locations: [{ type: 'url', target: 'https://github.com/a/a' }], + }, + }; + + const mockConfig = Object.assign(new ConfigReader(mutableConfigData), { + subscribe: (s: () => void) => { + subscriber = s; + return { unsubscribe: () => {} }; + }, + }); + + const mockConnection = { + applyMutation: jest.fn(), + } as unknown as EntityProviderConnection; + const locationProvider = new ConfigLocationEntityProvider(mockConfig); + + await locationProvider.connect(mockConnection); + + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + entity: expect.objectContaining({ + spec: { + target: 'https://github.com/a/a', + type: 'url', + }, + }), + locationKey: 'url:https://github.com/a/a', + }, + ], + }); + + mutableConfigData.catalog.locations[0].target = 'https://github.com/b/b'; + subscriber!(); + + expect(mockConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + entity: expect.objectContaining({ + spec: { + target: 'https://github.com/b/b', + type: 'url', + }, + }), + locationKey: 'url:https://github.com/b/b', + }, + ], + }); + }); }); diff --git a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts index 71ec84dae7..5deca09944 100644 --- a/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts @@ -21,8 +21,6 @@ import { EntityProvider, EntityProviderConnection } from './types'; import { locationSpecToLocationEntity } from './util'; export class ConfigLocationEntityProvider implements EntityProvider { - private connection: EntityProviderConnection | undefined; - constructor(private readonly config: Config) {} getProviderName(): string { @@ -30,12 +28,35 @@ export class ConfigLocationEntityProvider implements EntityProvider { } async connect(connection: EntityProviderConnection): Promise { - this.connection = connection; + const entities = this.getEntitiesFromConfig(); + await connection.applyMutation({ + type: 'full', + entities, + }); + if (this.config.subscribe) { + let currentKey = JSON.stringify(entities); + + this.config.subscribe(() => { + const newEntities = this.getEntitiesFromConfig(); + const newKey = JSON.stringify(newEntities); + + if (currentKey !== newKey) { + currentKey = newKey; + connection.applyMutation({ + type: 'full', + entities: newEntities, + }); + } + }); + } + } + + private getEntitiesFromConfig() { const locationConfigs = this.config.getOptionalConfigArray('catalog.locations') ?? []; - const entities = locationConfigs.map(location => { + return locationConfigs.map(location => { const type = location.getString('type'); const target = location.getString('target'); const entity = locationSpecToLocationEntity({ @@ -45,10 +66,5 @@ export class ConfigLocationEntityProvider implements EntityProvider { const locationKey = getEntityLocationRef(entity); return { entity, locationKey }; }); - - await this.connection.applyMutation({ - type: 'full', - entities, - }); } }