From a30cd576b14d9a65fcd3d161fef9c15ed4006a7e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 4 Oct 2020 14:44:22 +0200 Subject: [PATCH] config-loader: add support for new short-form secret syntax --- packages/config-loader/src/lib/reader.test.ts | 25 ++++++++++++++- packages/config-loader/src/lib/reader.ts | 19 +++++++++++- .../config-loader/src/lib/secrets.test.ts | 20 +++++++++--- packages/config-loader/src/lib/secrets.ts | 31 +++++++++---------- packages/config-loader/src/loader.test.ts | 3 +- 5 files changed, 74 insertions(+), 24 deletions(-) diff --git a/packages/config-loader/src/lib/reader.test.ts b/packages/config-loader/src/lib/reader.test.ts index 238a556061..3e6269b8d9 100644 --- a/packages/config-loader/src/lib/reader.test.ts +++ b/packages/config-loader/src/lib/reader.test.ts @@ -84,6 +84,29 @@ describe('readConfigFile', () => { }); it('should read secrets', async () => { + const readFile = memoryFiles({ + './app-config.yaml': 'app: { $file: "./my-secret" }', + }); + const readSecret = jest.fn().mockResolvedValue('secret'); + + const config = readConfigFile('./app-config.yaml', { + ...mockContext, + readFile, + readSecret: readSecret as ReadSecretFunc, + }); + + await expect(config).resolves.toEqual({ + data: { + app: 'secret', + }, + context: 'app-config.yaml', + }); + expect(readSecret).toHaveBeenCalledWith('.app', { + file: './my-secret', + }); + }); + + it('should read deprecated secrets', async () => { const readFile = memoryFiles({ './app-config.yaml': 'app: { $secret: { file: "./my-secret" } }', }); @@ -106,7 +129,7 @@ describe('readConfigFile', () => { }); }); - it('should require secrets to be objects', async () => { + it('should require deprecated secrets to be objects', async () => { const readFile = memoryFiles({ './app-config.yaml': 'app: { $secret: ["wrong-type"] }', }); diff --git a/packages/config-loader/src/lib/reader.ts b/packages/config-loader/src/lib/reader.ts index e2c9bebdf3..af4c63c83d 100644 --- a/packages/config-loader/src/lib/reader.ts +++ b/packages/config-loader/src/lib/reader.ts @@ -31,6 +31,8 @@ export async function readConfigFile( const configYaml = await ctx.readFile(filePath); const config = yaml.parse(configYaml); + const context = basename(filePath); + async function transform( obj: JsonValue, path: string, @@ -56,7 +58,11 @@ export async function readConfigFile( return arr; } + // TODO(Rugvip): This form of declaring secrets is deprecated, warn and remove in the future if ('$secret' in obj) { + console.warn( + `Deprecated secret declaration at '${path}' in '${context}', use $env, $file, etc. instead`, + ); if (!isObject(obj.$secret)) { throw TypeError(`Expected object at secret ${path}.$secret`); } @@ -68,6 +74,17 @@ export async function readConfigFile( } } + const [secretKey] = Object.keys(obj).filter(key => key.startsWith('$')); + if (secretKey) { + try { + return await ctx.readSecret(path, { + [secretKey.slice(1)]: obj[secretKey], + }); + } catch (error) { + throw new Error(`Invalid secret at ${path}: ${error.message}`); + } + } + const out: JsonObject = {}; for (const [key, value] of Object.entries(obj)) { @@ -87,5 +104,5 @@ export async function readConfigFile( if (!isObject(finalConfig)) { throw new TypeError('Expected object at config root'); } - return { data: finalConfig, context: basename(filePath) }; + return { data: finalConfig, context }; } diff --git a/packages/config-loader/src/lib/secrets.test.ts b/packages/config-loader/src/lib/secrets.test.ts index fd95e527af..cfc4150d68 100644 --- a/packages/config-loader/src/lib/secrets.test.ts +++ b/packages/config-loader/src/lib/secrets.test.ts @@ -56,21 +56,33 @@ describe('readSecret', () => { }); it('should read data secrets', async () => { + // Deprecated object form await expect( readSecret({ data: 'my-data.json', path: 'a.b.c' }, ctx), ).resolves.toBe('42'); - await expect( readSecret({ data: 'my-data.yaml', path: 'some.yaml.key' }, ctx), ).resolves.toBe('7'); - await expect( readSecret({ data: 'my-data.yml', path: 'different.key' }, ctx), ).resolves.toBe('hello'); - await expect( readSecret({ data: 'no-data.yml', path: 'different.key' }, ctx), ).rejects.toThrow('File not found!'); + + // New format with path in fragment + await expect(readSecret({ data: 'my-data.json#a.b.c' }, ctx)).resolves.toBe( + '42', + ); + await expect( + readSecret({ data: 'my-data.yaml#some.yaml.key' }, ctx), + ).resolves.toBe('7'); + await expect( + readSecret({ data: 'my-data.yml#different.key' }, ctx), + ).resolves.toBe('hello'); + await expect( + readSecret({ data: 'no-data.yml#different.key' }, ctx), + ).rejects.toThrow('File not found!'); }); it('should reject invalid secrets', async () => { @@ -84,7 +96,7 @@ describe('readSecret', () => { "Secret must contain one of 'file', 'env', 'data'", ); await expect(readSecret({ data: 'no-data.yml' }, ctx)).rejects.toThrow( - 'path is a required field', + "Invalid format for data secret value, must be of the form #, got 'no-data.yml'", ); await expect( readSecret({ data: 'no-parser.js', path: '.' }, ctx), diff --git a/packages/config-loader/src/lib/secrets.ts b/packages/config-loader/src/lib/secrets.ts index 348a41db58..4c3812d527 100644 --- a/packages/config-loader/src/lib/secrets.ts +++ b/packages/config-loader/src/lib/secrets.ts @@ -38,9 +38,8 @@ type EnvSecret = { type DataSecret = { // Path to the data secret file, relative to the config file. data: string; - // The path to the value inside the data file. - // Either a '.' separated list, or an array of path segments. - path: string | string[]; + // The path to the value inside the data file, each element separated by '.'. + path?: string; }; type Secret = FileSecret | EnvSecret | DataSecret; @@ -55,12 +54,6 @@ const secretLoaderSchemas = { }), data: yup.object({ data: yup.string().required(), - path: yup.lazy(value => { - if (typeof value === 'string') { - return yup.string().required(); - } - return yup.array().of(yup.string().required()).required(); - }), }), }; @@ -111,24 +104,30 @@ export async function readSecret( return ctx.env[secret.env]; } if ('data' in secret) { - const ext = extname(secret.data); + const url = + 'path' in secret ? `${secret.data}#${secret.path}` : secret.data; + const [filePath, dataPath] = url.split(/#(.*)/); + if (!dataPath) { + throw new Error( + `Invalid format for data secret value, must be of the form #, got '${url}'`, + ); + } + + const ext = extname(filePath); const parser = dataSecretParser[ext]; if (!parser) { throw new Error(`No data secret parser available for extension ${ext}`); } - const content = await ctx.readFile(secret.data); + const content = await ctx.readFile(filePath); - const { path } = secret; - const parts = typeof path === 'string' ? path.split('.') : path; + const parts = dataPath.split('.'); let value: JsonValue | undefined = await parser(content); for (const [index, part] of parts.entries()) { if (!isObject(value)) { const errPath = parts.slice(0, index).join('.'); - throw new Error( - `Value is not an object at ${errPath} in ${secret.data}`, - ); + throw new Error(`Value is not an object at ${errPath} in ${filePath}`); } value = value[part]; } diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index a7547ee8fb..087e2309c0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -24,8 +24,7 @@ describe('loadConfig', () => { app: title: Example App sessionKey: - $secret: - file: secrets/session-key.txt + $file: secrets/session-key.txt `, '/root/app-config.development.yaml': ` app: