Merge pull request #6754 from backstage/rugvip/watch

config-loader: enable config file watching and use to enable live reloading of catalog locations

.changeset/great-lemons-mix.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Enabled live reload of locations configured in `catalog.locations`.

.changeset/modern-ladybugs-promise.md

@@ -0,0 +1,5 @@
+---
+'@backstage/config': patch
+---
+
+Extended the `Config` interface to have an optional `subscribe` method that can be used be notified of updates to the configuration.

.changeset/thin-bugs-compete.md

@@ -0,0 +1,5 @@
+---
+'@backstage/config-loader': patch
+---
+
+Add support for config file watching through a new group of `watch` options to `loadConfig`.

.changeset/tricky-starfishes-cheer.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Add support for watching configuration by implementing the `subscribe` method in the configuration returned by `loadBackendConfig`.

packages/backend-common/src/config.ts

@@ -18,32 +18,142 @@ import { resolve as resolvePath } from 'path';
 import parseArgs from 'minimist';
 import { Logger } from 'winston';
 import { findPaths } from '@backstage/cli-common';
-import { Config, ConfigReader } from '@backstage/config';
+import { Config, ConfigReader, JsonValue } from '@backstage/config';
 import { loadConfig } from '@backstage/config-loader';
 
+class ObservableConfigProxy implements Config {
+  private config: Config = new ConfigReader({});
+
+  private readonly subscribers: (() => void)[] = [];
+
+  constructor(private readonly logger: Logger) {}
+
+  setConfig(config: Config) {
+    this.config = config;
+    for (const subscriber of this.subscribers) {
+      try {
+        subscriber();
+      } catch (error) {
+        this.logger.error(`Config subscriber threw error, ${error}`);
+      }
+    }
+  }
+
+  subscribe(onChange: () => void): { unsubscribe: () => void } {
+    this.subscribers.push(onChange);
+    return {
+      unsubscribe: () => {
+        const index = this.subscribers.indexOf(onChange);
+        if (index >= 0) {
+          this.subscribers.splice(index, 1);
+        }
+      },
+    };
+  }
+
+  has(key: string): boolean {
+    return this.config.has(key);
+  }
+  keys(): string[] {
+    return this.config.keys();
+  }
+  get<T = JsonValue>(key?: string): T {
+    return this.config.get(key);
+  }
+  getOptional<T = JsonValue>(key?: string): T | undefined {
+    return this.config.getOptional(key);
+  }
+  getConfig(key: string): Config {
+    return this.config.getConfig(key);
+  }
+  getOptionalConfig(key: string): Config | undefined {
+    return this.config.getOptionalConfig(key);
+  }
+  getConfigArray(key: string): Config[] {
+    return this.config.getConfigArray(key);
+  }
+  getOptionalConfigArray(key: string): Config[] | undefined {
+    return this.config.getOptionalConfigArray(key);
+  }
+  getNumber(key: string): number {
+    return this.config.getNumber(key);
+  }
+  getOptionalNumber(key: string): number | undefined {
+    return this.config.getOptionalNumber(key);
+  }
+  getBoolean(key: string): boolean {
+    return this.config.getBoolean(key);
+  }
+  getOptionalBoolean(key: string): boolean | undefined {
+    return this.config.getOptionalBoolean(key);
+  }
+  getString(key: string): string {
+    return this.config.getString(key);
+  }
+  getOptionalString(key: string): string | undefined {
+    return this.config.getOptionalString(key);
+  }
+  getStringArray(key: string): string[] {
+    return this.config.getStringArray(key);
+  }
+  getOptionalStringArray(key: string): string[] | undefined {
+    return this.config.getOptionalStringArray(key);
+  }
+}
+
 type Options = {
   logger: Logger;
   // process.argv or any other overrides
   argv: string[];
 };
 
+// A global used to ensure that only a single file watcher is active at a time.
+let currentCancelFunc: () => void;
+
 /**
- * Load configuration for a Backend
+ * Load configuration for a Backend.
+ *
+ * This function should only be called once, during the initialization of the backend.
  */
 export async function loadBackendConfig(options: Options): Promise<Config> {
   const args = parseArgs(options.argv);
-  const configOpts: string[] = [args.config ?? []].flat();
+  const configPaths: string[] = [args.config ?? []].flat();
+
+  const config = new ObservableConfigProxy(options.logger);
 
   /* eslint-disable-next-line no-restricted-syntax */
   const paths = findPaths(__dirname);
+
   const configs = await loadConfig({
     configRoot: paths.targetRoot,
-    configPaths: configOpts.map(opt => resolvePath(opt)),
+    configPaths: configPaths.map(opt => resolvePath(opt)),
+    watch: {
+      onChange(newConfigs) {
+        options.logger.info(
+          `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`,
+        );
+
+        config.setConfig(ConfigReader.fromConfigs(newConfigs));
+      },
+      stopSignal: new Promise(resolve => {
+        if (currentCancelFunc) {
+          currentCancelFunc();
+        }
+        currentCancelFunc = resolve;
+
+        // For reloads of this module we need to use a dispose handler rather than the global.
+        if (module.hot) {
+          module.hot.addDisposeHandler(resolve);
+        }
+      }),
+    },
   });
 
   options.logger.info(
     `Loaded config from ${configs.map(c => c.context).join(', ')}`,
   );
 
-  return ConfigReader.fromConfigs(configs);
+  config.setConfig(ConfigReader.fromConfigs(configs));
+
+  return config;
 }

packages/config-loader/api-report.md

@@ -37,6 +37,10 @@ export type LoadConfigOptions = {
   configPaths: string[];
   env?: string;
   experimentalEnvFunc?: EnvFunc;
+  watch?: {
+    onChange: (configs: AppConfig[]) => void;
+    stopSignal?: Promise<void>;
+  };
 };
 
 // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts

packages/config-loader/package.json

@@ -34,6 +34,7 @@
     "@backstage/config": "^0.1.6",
     "@types/json-schema": "^7.0.6",
     "ajv": "^7.0.3",
+    "chokidar": "^3.5.2",
     "fs-extra": "9.1.0",
     "json-schema": "^0.3.0",
     "json-schema-merge-allof": "^0.8.1",

packages/config-loader/src/loader.test.ts

@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
+import { AppConfig } from '@backstage/config';
 import { loadConfig } from './loader';
 import mockFs from 'mock-fs';
+import fs from 'fs-extra';
 
 describe('loadConfig', () => {
-  beforeAll(() => {
+  beforeEach(() => {
     process.env.MY_SECRET = 'is-secret';
     process.env.SUBSTITUTE_ME = 'substituted';
 
@@ -63,7 +65,7 @@ describe('loadConfig', () => {
     });
   });
 
-  afterAll(() => {
+  afterEach(() => {
     mockFs.restore();
   });
 
@@ -170,4 +172,81 @@ describe('loadConfig', () => {
       },
     ]);
   });
+
+  it('watches config files', async () => {
+    const onChange = defer<AppConfig[]>();
+    const stopSignal = defer<void>();
+
+    await expect(
+      loadConfig({
+        configRoot: '/root',
+        configPaths: [],
+        watch: {
+          onChange: onChange.resolve,
+          stopSignal: stopSignal.promise,
+        },
+      }),
+    ).resolves.toEqual([
+      {
+        context: 'app-config.yaml',
+        data: {
+          app: {
+            title: 'Example App',
+            sessionKey: 'abc123',
+            escaped: '${Escaped}',
+          },
+        },
+      },
+    ]);
+
+    await fs.writeJson('/root/app-config.yaml', {
+      app: {
+        title: 'New Title',
+      },
+    });
+    await expect(onChange.promise).resolves.toEqual([
+      {
+        context: 'app-config.yaml',
+        data: {
+          app: {
+            title: 'New Title',
+          },
+        },
+      },
+    ]);
+
+    stopSignal.resolve();
+  });
+
+  it('stops watching config files', async () => {
+    const stopSignal = defer<void>();
+
+    await loadConfig({
+      configRoot: '/root',
+      configPaths: [],
+      watch: {
+        onChange: () => {
+          expect('not').toBe('called');
+        },
+        stopSignal: stopSignal.promise,
+      },
+    });
+
+    stopSignal.resolve();
+
+    await fs.writeJson('/root/app-config.yaml', {
+      app: {
+        title: 'New Title',
+      },
+    });
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  });
+
+  function defer<T>() {
+    let resolve: (value: T) => void;
+    const promise = new Promise<T>(_resolve => {
+      resolve = _resolve;
+    });
+    return { promise, resolve: resolve! };
+  }
 });

packages/config-loader/src/loader.ts

@@ -16,6 +16,7 @@
 
 import fs from 'fs-extra';
 import yaml from 'yaml';
+import chokidar from 'chokidar';
 import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path';
 import { AppConfig } from '@backstage/config';
 import {
@@ -42,13 +43,27 @@ export type LoadConfigOptions = {
    * @experimental This API is not stable and may change at any point
    */
   experimentalEnvFunc?: EnvFunc;
+
+  /**
+   * An optional configuration that enables watching of config files.
+   */
+  watch?: {
+    /**
+     * A listener that is called when a config file is changed.
+     */
+    onChange: (configs: AppConfig[]) => void;
+
+    /**
+     * An optional signal that stops the watcher once the promise resolves.
+     */
+    stopSignal?: Promise<void>;
+  };
 };
 
 export async function loadConfig(
   options: LoadConfigOptions,
 ): Promise<AppConfig[]> {
-  const configs = [];
-  const { configRoot, experimentalEnvFunc: envFunc } = options;
+  const { configRoot, experimentalEnvFunc: envFunc, watch } = options;
   const configPaths = options.configPaths.slice();
 
   // If no paths are provided, we default to reading
@@ -64,7 +79,9 @@ export async function loadConfig(
 
   const env = envFunc ?? (async (name: string) => process.env[name]);
 
-  try {
+  const loadConfigFiles = async () => {
+    const configs = [];
+
     for (const configPath of configPaths) {
       if (!isAbsolute(configPath)) {
         throw new Error(`Config load path is not absolute: '${configPath}'`);
@@ -83,13 +100,50 @@ export async function loadConfig(
 
       configs.push({ data, context: basename(configPath) });
     }
+
+    return configs;
+  };
+
+  let fileConfigs;
+  try {
+    fileConfigs = await loadConfigFiles();
   } catch (error) {
     throw new Error(
       `Failed to read static configuration file, ${error.message}`,
     );
   }
 
-  configs.push(...readEnvConfig(process.env));
+  const envConfigs = await readEnvConfig(process.env);
 
-  return configs;
+  // Set up config file watching if requested by the caller
+  if (watch) {
+    let currentSerializedConfig = JSON.stringify(fileConfigs);
+
+    const watcher = chokidar.watch(configPaths, {
+      usePolling: process.env.NODE_ENV === 'test',
+    });
+    watcher.on('change', async () => {
+      try {
+        const newConfigs = await loadConfigFiles();
+        const newSerializedConfig = JSON.stringify(newConfigs);
+
+        if (currentSerializedConfig === newSerializedConfig) {
+          return;
+        }
+        currentSerializedConfig = newSerializedConfig;
+
+        watch.onChange([...newConfigs, ...envConfigs]);
+      } catch (error) {
+        console.error(`Failed to reload configuration files, ${error}`);
+      }
+    });
+
+    if (watch.stopSignal) {
+      watch.stopSignal.then(() => {
+        watcher.close();
+      });
+    }
+  }
+
+  return [...fileConfigs, ...envConfigs];
 }

packages/config/api-report.md

@@ -16,6 +16,9 @@ export type AppConfig = {
 //
 // @public (undocumented)
 export type Config = {
+  subscribe?(onChange: () => void): {
+    unsubscribe: () => void;
+  };
   has(key: string): boolean;
   keys(): string[];
   get<T = JsonValue>(key?: string): T;

packages/config/src/types.ts

@@ -26,6 +26,17 @@ export type AppConfig = {
 };
 
 export type Config = {
+  /**
+   * Subscribes to the configuration object in order to receive a notification
+   * whenever any value within the configuration has changed.
+   *
+   * This method is optional to implement, and consumers need to check if it is
+   * implemented before invoking it.
+   */
+  subscribe?(onChange: () => void): {
+    unsubscribe: () => void;
+  };
+
   has(key: string): boolean;
 
   keys(): string[];

plugins/catalog-backend/src/next/ConfigLocationEntityProvider.test.ts

@@ -70,4 +70,61 @@ describe('ConfigLocationEntityProvider', () => {
       ]),
     });
   });
+
+  it('should be able to observe the config', async () => {
+    // Grab the subscriber function and use mutable config data to mock a config file change
+    let subscriber: () => void;
+    const mutableConfigData = {
+      catalog: {
+        locations: [{ type: 'url', target: 'https://github.com/a/a' }],
+      },
+    };
+
+    const mockConfig = Object.assign(new ConfigReader(mutableConfigData), {
+      subscribe: (s: () => void) => {
+        subscriber = s;
+        return { unsubscribe: () => {} };
+      },
+    });
+
+    const mockConnection = {
+      applyMutation: jest.fn(),
+    } as unknown as EntityProviderConnection;
+    const locationProvider = new ConfigLocationEntityProvider(mockConfig);
+
+    await locationProvider.connect(mockConnection);
+
+    expect(mockConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: [
+        {
+          entity: expect.objectContaining({
+            spec: {
+              target: 'https://github.com/a/a',
+              type: 'url',
+            },
+          }),
+          locationKey: 'url:https://github.com/a/a',
+        },
+      ],
+    });
+
+    mutableConfigData.catalog.locations[0].target = 'https://github.com/b/b';
+    subscriber!();
+
+    expect(mockConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'full',
+      entities: [
+        {
+          entity: expect.objectContaining({
+            spec: {
+              target: 'https://github.com/b/b',
+              type: 'url',
+            },
+          }),
+          locationKey: 'url:https://github.com/b/b',
+        },
+      ],
+    });
+  });
 });

plugins/catalog-backend/src/next/ConfigLocationEntityProvider.ts

@@ -21,8 +21,6 @@ import { EntityProvider, EntityProviderConnection } from './types';
 import { locationSpecToLocationEntity } from './util';
 
 export class ConfigLocationEntityProvider implements EntityProvider {
-  private connection: EntityProviderConnection | undefined;
-
   constructor(private readonly config: Config) {}
 
   getProviderName(): string {
@@ -30,12 +28,35 @@ export class ConfigLocationEntityProvider implements EntityProvider {
   }
 
   async connect(connection: EntityProviderConnection): Promise<void> {
-    this.connection = connection;
+    const entities = this.getEntitiesFromConfig();
+    await connection.applyMutation({
+      type: 'full',
+      entities,
+    });
 
+    if (this.config.subscribe) {
+      let currentKey = JSON.stringify(entities);
+
+      this.config.subscribe(() => {
+        const newEntities = this.getEntitiesFromConfig();
+        const newKey = JSON.stringify(newEntities);
+
+        if (currentKey !== newKey) {
+          currentKey = newKey;
+          connection.applyMutation({
+            type: 'full',
+            entities: newEntities,
+          });
+        }
+      });
+    }
+  }
+
+  private getEntitiesFromConfig() {
     const locationConfigs =
       this.config.getOptionalConfigArray('catalog.locations') ?? [];
 
-    const entities = locationConfigs.map(location => {
+    return locationConfigs.map(location => {
       const type = location.getString('type');
       const target = location.getString('target');
       const entity = locationSpecToLocationEntity({
@@ -45,10 +66,5 @@ export class ConfigLocationEntityProvider implements EntityProvider {
       const locationKey = getEntityLocationRef(entity);
       return { entity, locationKey };
     });
-
-    await this.connection.applyMutation({
-      type: 'full',
-      entities,
-    });
   }
 }