config-loader: add support for new short-form secret syntax

packages/config-loader/src/lib/reader.test.ts

@@ -84,6 +84,29 @@ describe('readConfigFile', () => {
   });
 
   it('should read secrets', async () => {
+    const readFile = memoryFiles({
+      './app-config.yaml': 'app: { $file: "./my-secret" }',
+    });
+    const readSecret = jest.fn().mockResolvedValue('secret');
+
+    const config = readConfigFile('./app-config.yaml', {
+      ...mockContext,
+      readFile,
+      readSecret: readSecret as ReadSecretFunc,
+    });
+
+    await expect(config).resolves.toEqual({
+      data: {
+        app: 'secret',
+      },
+      context: 'app-config.yaml',
+    });
+    expect(readSecret).toHaveBeenCalledWith('.app', {
+      file: './my-secret',
+    });
+  });
+
+  it('should read deprecated secrets', async () => {
     const readFile = memoryFiles({
       './app-config.yaml': 'app: { $secret: { file: "./my-secret" } }',
     });
@@ -106,7 +129,7 @@ describe('readConfigFile', () => {
     });
   });
 
-  it('should require secrets to be objects', async () => {
+  it('should require deprecated secrets to be objects', async () => {
     const readFile = memoryFiles({
       './app-config.yaml': 'app: { $secret: ["wrong-type"] }',
     });

packages/config-loader/src/lib/reader.ts

@@ -31,6 +31,8 @@ export async function readConfigFile(
   const configYaml = await ctx.readFile(filePath);
   const config = yaml.parse(configYaml);
 
+  const context = basename(filePath);
+
   async function transform(
     obj: JsonValue,
     path: string,
@@ -56,7 +58,11 @@ export async function readConfigFile(
       return arr;
     }
 
+    // TODO(Rugvip): This form of declaring secrets is deprecated, warn and remove in the future
     if ('$secret' in obj) {
+      console.warn(
+        `Deprecated secret declaration at '${path}' in '${context}', use $env, $file, etc. instead`,
+      );
       if (!isObject(obj.$secret)) {
         throw TypeError(`Expected object at secret ${path}.$secret`);
       }
@@ -68,6 +74,17 @@ export async function readConfigFile(
       }
     }
 
+    const [secretKey] = Object.keys(obj).filter(key => key.startsWith('$'));
+    if (secretKey) {
+      try {
+        return await ctx.readSecret(path, {
+          [secretKey.slice(1)]: obj[secretKey],
+        });
+      } catch (error) {
+        throw new Error(`Invalid secret at ${path}: ${error.message}`);
+      }
+    }
+
     const out: JsonObject = {};
 
     for (const [key, value] of Object.entries(obj)) {
@@ -87,5 +104,5 @@ export async function readConfigFile(
   if (!isObject(finalConfig)) {
     throw new TypeError('Expected object at config root');
   }
-  return { data: finalConfig, context: basename(filePath) };
+  return { data: finalConfig, context };
 }

packages/config-loader/src/lib/secrets.test.ts

@@ -56,21 +56,33 @@ describe('readSecret', () => {
   });
 
   it('should read data secrets', async () => {
+    // Deprecated object form
     await expect(
       readSecret({ data: 'my-data.json', path: 'a.b.c' }, ctx),
     ).resolves.toBe('42');
-
     await expect(
       readSecret({ data: 'my-data.yaml', path: 'some.yaml.key' }, ctx),
     ).resolves.toBe('7');
-
     await expect(
       readSecret({ data: 'my-data.yml', path: 'different.key' }, ctx),
     ).resolves.toBe('hello');
-
     await expect(
       readSecret({ data: 'no-data.yml', path: 'different.key' }, ctx),
     ).rejects.toThrow('File not found!');
+
+    // New format with path in fragment
+    await expect(readSecret({ data: 'my-data.json#a.b.c' }, ctx)).resolves.toBe(
+      '42',
+    );
+    await expect(
+      readSecret({ data: 'my-data.yaml#some.yaml.key' }, ctx),
+    ).resolves.toBe('7');
+    await expect(
+      readSecret({ data: 'my-data.yml#different.key' }, ctx),
+    ).resolves.toBe('hello');
+    await expect(
+      readSecret({ data: 'no-data.yml#different.key' }, ctx),
+    ).rejects.toThrow('File not found!');
   });
 
   it('should reject invalid secrets', async () => {
@@ -84,7 +96,7 @@ describe('readSecret', () => {
       "Secret must contain one of 'file', 'env', 'data'",
     );
     await expect(readSecret({ data: 'no-data.yml' }, ctx)).rejects.toThrow(
-      'path is a required field',
+      "Invalid format for data secret value, must be of the form <filepath>#<datapath>, got 'no-data.yml'",
     );
     await expect(
       readSecret({ data: 'no-parser.js', path: '.' }, ctx),

packages/config-loader/src/lib/secrets.ts

@@ -38,9 +38,8 @@ type EnvSecret = {
 type DataSecret = {
   // Path to the data secret file, relative to the config file.
   data: string;
-  // The path to the value inside the data file.
-  // Either a '.' separated list, or an array of path segments.
-  path: string | string[];
+  // The path to the value inside the data file, each element separated by '.'.
+  path?: string;
 };
 
 type Secret = FileSecret | EnvSecret | DataSecret;
@@ -55,12 +54,6 @@ const secretLoaderSchemas = {
   }),
   data: yup.object({
     data: yup.string().required(),
-    path: yup.lazy(value => {
-      if (typeof value === 'string') {
-        return yup.string().required();
-      }
-      return yup.array().of(yup.string().required()).required();
-    }),
   }),
 };
 
@@ -111,24 +104,30 @@ export async function readSecret(
     return ctx.env[secret.env];
   }
   if ('data' in secret) {
-    const ext = extname(secret.data);
+    const url =
+      'path' in secret ? `${secret.data}#${secret.path}` : secret.data;
+    const [filePath, dataPath] = url.split(/#(.*)/);
+    if (!dataPath) {
+      throw new Error(
+        `Invalid format for data secret value, must be of the form <filepath>#<datapath>, got '${url}'`,
+      );
+    }
+
+    const ext = extname(filePath);
     const parser = dataSecretParser[ext];
     if (!parser) {
       throw new Error(`No data secret parser available for extension ${ext}`);
     }
 
-    const content = await ctx.readFile(secret.data);
+    const content = await ctx.readFile(filePath);
 
-    const { path } = secret;
-    const parts = typeof path === 'string' ? path.split('.') : path;
+    const parts = dataPath.split('.');
 
     let value: JsonValue | undefined = await parser(content);
     for (const [index, part] of parts.entries()) {
       if (!isObject(value)) {
         const errPath = parts.slice(0, index).join('.');
-        throw new Error(
-          `Value is not an object at ${errPath} in ${secret.data}`,
-        );
+        throw new Error(`Value is not an object at ${errPath} in ${filePath}`);
       }
       value = value[part];
     }

packages/config-loader/src/loader.test.ts

@@ -24,8 +24,7 @@ describe('loadConfig', () => {
         app:
           title: Example App
           sessionKey:
-            $secret:
-              file: secrets/session-key.txt
+            $file: secrets/session-key.txt
       `,
       '/root/app-config.development.yaml': `
         app: