Config: Add $include which replaces $data and can include any value

packages/config-loader/src/lib/secrets.test.ts

@@ -28,6 +28,7 @@ const ctx: ReaderContext = {
       'my-data.json': '{"a":{"b":{"c":42}}}',
       'my-data.yaml': 'some:\n yaml:\n  key: 7',
       'my-data.yml': 'different: { key: hello }',
+      'invalid.yaml': 'foo: [}',
     } as { [key: string]: string })[path];
 
     if (!content) {
@@ -84,6 +85,41 @@ describe('readSecret', () => {
     ).rejects.toThrow('File not found!');
   });
 
+  it('should include extra files', async () => {
+    // New format with path in fragment
+    await expect(
+      readSecret({ include: 'my-data.json#a.b.c' }, ctx),
+    ).resolves.toBe(42);
+    await expect(
+      readSecret({ include: 'my-data.json#a.b' }, ctx),
+    ).resolves.toEqual({ c: 42 });
+    await expect(
+      readSecret({ include: 'my-data.yaml#some.yaml.key' }, ctx),
+    ).resolves.toBe(7);
+    await expect(readSecret({ include: 'my-data.yaml' }, ctx)).resolves.toEqual(
+      {
+        some: { yaml: { key: 7 } },
+      },
+    );
+    await expect(
+      readSecret({ include: 'my-data.yaml#' }, ctx),
+    ).resolves.toEqual({
+      some: { yaml: { key: 7 } },
+    });
+    await expect(
+      readSecret({ include: 'my-data.yml#different.key' }, ctx),
+    ).resolves.toBe('hello');
+    await expect(
+      readSecret({ include: 'no-data.yml#different.key' }, ctx),
+    ).rejects.toThrow('File not found!');
+    await expect(
+      readSecret({ include: 'my-data.yml#missing.key' }, ctx),
+    ).rejects.toThrow('Value is not an object at missing in my-data.yml');
+    await expect(readSecret({ include: 'invalid.yaml' }, ctx)).rejects.toThrow(
+      'Failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }',
+    );
+  });
+
   it('should reject invalid secrets', async () => {
     await expect(readSecret('hello' as any, ctx)).rejects.toThrow(
       'secret must be a `object` type, but the final value was: `"hello"`.',

packages/config-loader/src/lib/secrets.ts

@@ -55,6 +55,9 @@ const secretLoaderSchemas = {
   data: yup.object({
     data: yup.string().required(),
   }),
+  include: yup.object({
+    include: yup.string().required(),
+  }),
 };
 
 // The top-level secret schema, which figures out what type of secret it is.
@@ -94,7 +97,7 @@ const dataSecretParser: {
 export async function readSecret(
   data: JsonObject,
   ctx: ReaderContext,
-): Promise<string | undefined> {
+): Promise<JsonValue | undefined> {
   const secret = secretSchema.validateSync(data, { strict: true }) as Secret;
 
   if ('file' in secret) {
@@ -134,6 +137,35 @@ export async function readSecret(
 
     return String(value);
   }
+  if ('include' in secret) {
+    const [filePath, dataPath] = secret.include.split(/#(.*)/);
+
+    const ext = extname(filePath);
+    const parser = dataSecretParser[ext];
+    if (!parser) {
+      throw new Error(`No data secret parser available for extension ${ext}`);
+    }
+
+    const content = await ctx.readFile(filePath);
+
+    const parts = dataPath ? dataPath.split('.') : [];
+
+    let value: JsonValue | undefined;
+    try {
+      value = await parser(content);
+    } catch (error) {
+      throw new Error(`Failed to parse included file ${filePath}, ${error}`);
+    }
+    for (const [index, part] of parts.entries()) {
+      if (!isObject(value)) {
+        const errPath = parts.slice(0, index).join('.');
+        throw new Error(`Value is not an object at ${errPath} in ${filePath}`);
+      }
+      value = value[part];
+    }
+
+    return value;
+  }
 
   isNever<typeof secret>();
   throw new Error('Secret was left unhandled');

packages/config-loader/src/lib/types.ts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-import { JsonObject } from '@backstage/config';
+import { JsonObject, JsonValue } from '@backstage/config';
 
 export type ReadFileFunc = (path: string) => Promise<string>;
 export type ReadSecretFunc = (
   path: string,
   desc: JsonObject,
-) => Promise<string | undefined>;
+) => Promise<JsonValue | undefined>;
 export type SkipFunc = (path: string) => boolean;
 
 /**

packages/config-loader/src/loader.ts

@@ -16,7 +16,7 @@
 
 import fs from 'fs-extra';
 import { resolve as resolvePath, dirname, isAbsolute } from 'path';
-import { AppConfig, JsonObject } from '@backstage/config';
+import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
 import { readConfigFile, readEnvConfig, readSecret } from './lib';
 
 export type LoadConfigOptions = {
@@ -49,7 +49,7 @@ class Context {
   async readSecret(
     _path: string,
     desc: JsonObject,
-  ): Promise<string | undefined> {
+  ): Promise<JsonValue | undefined> {
     return readSecret(desc, this);
   }
 }