Merge pull request #9608 from backstage/watch-additional-files

config-loader: watch any extra files loaded when loading local config

.changeset/pink-fans-tie.md

@@ -0,0 +1,5 @@
+---
+'@backstage/config-loader': patch
+---
+
+Include any files included in configuration via $include or $file directives when watching for configuration changes.

packages/config-loader/src/loader.test.ts

@@ -333,6 +333,66 @@ describe('loadConfig', () => {
     stopSignal.resolve();
   });
 
+  it('watches included files', async () => {
+    const onChange = defer<AppConfig[]>();
+    const stopSignal = defer<void>();
+
+    await expect(
+      loadConfig({
+        configRoot: '/root',
+        configTargets: [{ path: '/root/app-config.development.yaml' }],
+        watch: {
+          onChange: onChange.resolve,
+          stopSignal: stopSignal.promise,
+        },
+      }),
+    ).resolves.toEqual({
+      appConfigs: [
+        {
+          context: 'app-config.development.yaml',
+          data: {
+            app: {
+              sessionKey: 'development-key',
+            },
+            backend: {
+              foo: {
+                bar: 'token is-secret',
+              },
+            },
+            other: {
+              secret: 'abc123',
+            },
+          },
+        },
+      ],
+    });
+
+    // session-key is indirectly included in app-config.development.yaml
+    // via included.yaml
+    await fs.writeFile('/root/secrets/session-key.txt', 'abc234');
+
+    await expect(onChange.promise).resolves.toEqual([
+      {
+        context: 'app-config.development.yaml',
+        data: {
+          app: {
+            sessionKey: 'development-key',
+          },
+          backend: {
+            foo: {
+              bar: 'token is-secret',
+            },
+          },
+          other: {
+            secret: 'abc234',
+          },
+        },
+      },
+    ]);
+
+    stopSignal.resolve();
+  });
+
   it('watches remote config urls', async () => {
     server.use(initialLoaderHandler);
 

packages/config-loader/src/loader.ts

@@ -140,7 +140,8 @@ export async function loadConfig(
   const env = envFunc ?? (async (name: string) => process.env[name]);
 
   const loadConfigFiles = async () => {
-    const configs = [];
+    const fileConfigs = [];
+    const loadedPaths = new Set<string>();
 
     for (const configPath of configPaths) {
       if (!isAbsolute(configPath)) {
@@ -148,8 +149,15 @@ export async function loadConfig(
       }
 
       const dir = dirname(configPath);
-      const readFile = (path: string) =>
-        fs.readFile(resolvePath(dir, path), 'utf8');
+      const readFile = (path: string) => {
+        const fullPath = resolvePath(dir, path);
+        // if we read a file when building configuration,
+        // we should include that file when watching for
+        // changes, too.
+        loadedPaths.add(fullPath);
+
+        return fs.readFile(fullPath, 'utf8');
+      };
 
       const input = yaml.parse(await readFile(configPath));
       const substitutionTransform = createSubstitutionTransform(env);
@@ -158,10 +166,10 @@ export async function loadConfig(
         substitutionTransform,
       ]);
 
-      configs.push({ data, context: basename(configPath) });
+      fileConfigs.push({ data, context: basename(configPath) });
     }
 
-    return configs;
+    return { fileConfigs, loadedPaths };
   };
 
   const loadRemoteConfigFiles = async () => {
@@ -199,8 +207,9 @@ export async function loadConfig(
   };
 
   let fileConfigs: AppConfig[];
+  let loadedPaths: Set<string>;
   try {
-    fileConfigs = await loadConfigFiles();
+    ({ fileConfigs, loadedPaths } = await loadConfigFiles());
   } catch (error) {
     throw new ForwardedError('Failed to read static configuration file', error);
   }
@@ -220,14 +229,24 @@ export async function loadConfig(
   const envConfigs = await readEnvConfig(process.env);
 
   const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => {
-    const watcher = chokidar.watch(configPaths, {
+    let watchedFiles = Array.from(loadedPaths);
+
+    const watcher = chokidar.watch(watchedFiles, {
       usePolling: process.env.NODE_ENV === 'test',
     });
 
     let currentSerializedConfig = JSON.stringify(fileConfigs);
     watcher.on('change', async () => {
       try {
-        const newConfigs = await loadConfigFiles();
+        const { fileConfigs: newConfigs, loadedPaths: newLoadedPaths } =
+          await loadConfigFiles();
+
+        // Replace watches to handle any added or removed
+        // $include or $file expressions.
+        watcher.unwatch(watchedFiles);
+        watchedFiles = Array.from(newLoadedPaths);
+        watcher.add(watchedFiles);
+
         const newSerializedConfig = JSON.stringify(newConfigs);
 
         if (currentSerializedConfig === newSerializedConfig) {