From 26a94cfac6468da1fb6a261401716a34b9ba5b65 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 23 Jan 2021 18:59:45 +0100 Subject: [PATCH] config-loader: refactor to use transforms as a base building block --- .../config-loader/src/lib/include.test.ts | 134 +++++++++++++++++ packages/config-loader/src/lib/include.ts | 117 +++++++++++++++ packages/config-loader/src/lib/index.ts | 4 +- packages/config-loader/src/lib/reader.test.ts | 140 ------------------ .../config-loader/src/lib/secrets.test.ts | 129 ---------------- packages/config-loader/src/lib/secrets.ts | 134 ----------------- .../config-loader/src/lib/transform.test.ts | 82 ++++++++++ .../src/lib/{reader.ts => transform.ts} | 64 ++++---- packages/config-loader/src/lib/types.ts | 20 +-- packages/config-loader/src/lib/utils.ts | 5 - packages/config-loader/src/loader.ts | 57 +++---- 11 files changed, 390 insertions(+), 496 deletions(-) create mode 100644 packages/config-loader/src/lib/include.test.ts create mode 100644 packages/config-loader/src/lib/include.ts delete mode 100644 packages/config-loader/src/lib/reader.test.ts delete mode 100644 packages/config-loader/src/lib/secrets.test.ts delete mode 100644 packages/config-loader/src/lib/secrets.ts create mode 100644 packages/config-loader/src/lib/transform.test.ts rename packages/config-loader/src/lib/{reader.ts => transform.ts} (59%) diff --git a/packages/config-loader/src/lib/include.test.ts b/packages/config-loader/src/lib/include.test.ts new file mode 100644 index 0000000000..b25b601cc9 --- /dev/null +++ b/packages/config-loader/src/lib/include.test.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createIncludeTransform } from './include'; + +const env = jest.fn(async (name: string) => { + return ({ + SECRET: 'my-secret', + } as { [name: string]: string })[name]; +}); + +const readFile = jest.fn(async (path: string) => { + const content = ({ + 'my-secret': 'secret', + 'my-data.json': '{"a":{"b":{"c":42}}}', + 'my-data.yaml': 'some:\n yaml:\n key: 7', + 'my-data.yml': 'different: { key: hello }', + 'invalid.yaml': 'foo: [}', + } as { [key: string]: string })[path]; + + if (!content) { + throw new Error('File not found!'); + } + return content; +}); + +const includeTransform = createIncludeTransform(env, readFile); + +describe('includeTransform', () => { + it('should not transform unknown values', async () => { + await expect(includeTransform('foo')).resolves.toEqual([ + false, + expect.anything(), + ]); + await expect(includeTransform([1])).resolves.toEqual([ + false, + expect.anything(), + ]); + await expect(includeTransform(1)).resolves.toEqual([ + false, + expect.anything(), + ]); + await expect(includeTransform({ x: 'y' })).resolves.toEqual([ + false, + expect.anything(), + ]); + await expect(includeTransform(null)).resolves.toEqual([false, null]); + }); + + it('should include text files', async () => { + await expect(includeTransform({ $file: 'my-secret' })).resolves.toEqual([ + true, + 'secret', + ]); + await expect(includeTransform({ $file: 'no-secret' })).rejects.toThrow( + 'File not found!', + ); + }); + + it('should include env vars', async () => { + await expect(includeTransform({ $env: 'SECRET' })).resolves.toEqual([ + true, + 'my-secret', + ]); + await expect(includeTransform({ $env: 'NO_SECRET' })).resolves.toEqual([ + true, + undefined, + ]); + }); + + it('should include config files', async () => { + // New format with path in fragment + await expect( + includeTransform({ $include: 'my-data.json#a.b.c' }), + ).resolves.toEqual([true, 42]); + await expect( + includeTransform({ $include: 'my-data.json#a.b' }), + ).resolves.toEqual([true, { c: 42 }]); + await expect( + includeTransform({ $include: 'my-data.yaml#some.yaml.key' }), + ).resolves.toEqual([true, 7]); + await expect( + includeTransform({ $include: 'my-data.yaml' }), + ).resolves.toEqual([ + true, + { + some: { yaml: { key: 7 } }, + }, + ]); + await expect( + includeTransform({ $include: 'my-data.yaml#' }), + ).resolves.toEqual([ + true, + { + some: { yaml: { key: 7 } }, + }, + ]); + await expect( + includeTransform({ $include: 'my-data.yml#different.key' }), + ).resolves.toEqual([true, 'hello']); + }); + + it('should reject invalid includes', async () => { + await expect( + includeTransform({ $include: 'no-parser.js' }), + ).rejects.toThrow('no configuration parser available for extension .js'); + await expect( + includeTransform({ $include: 'no-data.yml#different.key' }), + ).rejects.toThrow('File not found!'); + await expect( + includeTransform({ $include: 'my-data.yml#missing.key' }), + ).rejects.toThrow( + "value at 'missing' in included file my-data.yml is not an object", + ); + await expect( + includeTransform({ $include: 'invalid.yaml' }), + ).rejects.toThrow( + 'failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }', + ); + }); +}); diff --git a/packages/config-loader/src/lib/include.ts b/packages/config-loader/src/lib/include.ts new file mode 100644 index 0000000000..c08da09e7a --- /dev/null +++ b/packages/config-loader/src/lib/include.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { extname } from 'path'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { isObject } from './utils'; +import { TransformFunc, EnvFunc, ReadFileFunc } from './types'; + +// Parsers for each type of included file +const includeFileParser: { + [ext in string]: (content: string) => Promise; +} = { + '.json': async content => JSON.parse(content), + '.yaml': async content => yaml.parse(content), + '.yml': async content => yaml.parse(content), +}; + +/** + * Transforms a secret description into the actual secret value. + */ +export function createIncludeTransform( + env: EnvFunc, + readFile: ReadFileFunc, +): TransformFunc { + return async (input: JsonValue) => { + if (!isObject(input)) { + return [false, input]; + } + // Check if there's any key that starts with a '$', in that case we treat + // this entire object as a secret. + const [secretKey] = Object.keys(input).filter(key => key.startsWith('$')); + if (secretKey) { + if (Object.keys(input).length !== 1) { + throw new Error( + `include key ${secretKey} should not have adjacent keys`, + ); + } + } else { + return [false, input]; + } + + const secretValue = input[secretKey]; + if (typeof secretValue !== 'string') { + throw new Error(`${secretKey} include value is not a string`); + } + + switch (secretKey) { + case '$file': + try { + return [true, await readFile(secretValue)]; + } catch (error) { + throw new Error(`failed to read file ${secretValue}, ${error}`); + } + case '$env': + try { + return [true, await env(secretValue)]; + } catch (error) { + throw new Error(`failed to read env ${secretValue}, ${error}`); + } + + case '$include': { + const [filePath, dataPath] = secretValue.split(/#(.*)/); + + const ext = extname(filePath); + const parser = includeFileParser[ext]; + if (!parser) { + throw new Error( + `no configuration parser available for included file ${filePath}`, + ); + } + + const content = await readFile(filePath); + + const parts = dataPath ? dataPath.split('.') : []; + + let value: JsonValue | undefined; + try { + value = await parser(content); + } catch (error) { + throw new Error( + `failed to parse included file ${filePath}, ${error}`, + ); + } + + // This bit handles selecting a subtree in the included file, if a path was provided after a # + for (const [index, part] of parts.entries()) { + if (!isObject(value)) { + const errPath = parts.slice(0, index).join('.'); + throw new Error( + `value at '${errPath}' in included file ${filePath} is not an object`, + ); + } + value = value[part]; + } + + return [true, value]; + } + + default: + throw new Error(`unknown secret ${secretKey}`); + } + }; +} diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index ceb7c34222..90e64546f6 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { readConfigFile } from './reader'; +export { applyConfigTransforms } from './transform'; export { readEnvConfig } from './env'; -export { readSecret } from './secrets'; +export { createIncludeTransform } from './include'; export * from './schema'; diff --git a/packages/config-loader/src/lib/reader.test.ts b/packages/config-loader/src/lib/reader.test.ts deleted file mode 100644 index 0a3fe01bf7..0000000000 --- a/packages/config-loader/src/lib/reader.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { readConfigFile } from './reader'; -import { ReaderContext, ReadSecretFunc } from './types'; - -function memoryFiles(files: { [path: string]: string }) { - return async (path: string) => { - if (path in files) { - return files[path]; - } - throw new Error(`File not found, ${path}`); - }; -} - -const mockContext: ReaderContext = { - env: {}, - readFile: jest.fn(), - readSecret: jest.fn(), -}; - -describe('readConfigFile', () => { - it('should read a plain config file', async () => { - const readFile = memoryFiles({ - './app-config.yaml': - 'app: { title: "Test", x: 1, y: [null, true], z: null }', - }); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - }); - - await expect(config).resolves.toEqual({ - data: { - app: { - title: 'Test', - x: 1, - y: [true], - }, - }, - context: 'app-config.yaml', - }); - }); - - it('should error out if the config file has invalid syntax', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { title: ]', - }); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - }); - - await expect(config).rejects.toThrow('Flow map contains an unexpected ]'); - }); - - it('should error out if config is not an object', async () => { - const readFile = memoryFiles({ - './app-config.yaml': '[]', - }); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - }); - - await expect(config).rejects.toThrow('Expected object at config root'); - }); - - it('should read secrets', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { $file: "./my-secret" }', - }); - const readSecret = jest.fn().mockResolvedValue('secret'); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - readSecret: readSecret as ReadSecretFunc, - }); - - await expect(config).resolves.toEqual({ - data: { - app: 'secret', - }, - context: 'app-config.yaml', - }); - expect(readSecret).toHaveBeenCalledWith('.app', { - file: './my-secret', - }); - }); - - it('should not allow keys adjacent to secrets', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { extraKey: 3, $file: "./my-secret" }', - }); - const readSecret = jest.fn().mockResolvedValue('secret'); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - readSecret: readSecret as ReadSecretFunc, - }); - - await expect(config).rejects.toThrow( - "Secret key '$file' has adjacent keys at .app", - ); - expect(readSecret).not.toHaveBeenCalled(); - }); - - it('should forward secret reading errors', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { $file: {} }', - }); - const readSecret = jest.fn().mockRejectedValue(new Error('NOPE')); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - readSecret: readSecret as ReadSecretFunc, - }); - - await expect(config).rejects.toThrow('Invalid secret at .app: NOPE'); - }); -}); diff --git a/packages/config-loader/src/lib/secrets.test.ts b/packages/config-loader/src/lib/secrets.test.ts deleted file mode 100644 index 5dd06a9680..0000000000 --- a/packages/config-loader/src/lib/secrets.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { readSecret } from './secrets'; -import { ReaderContext } from './types'; - -const ctx: ReaderContext = { - env: { - SECRET: 'my-secret', - }, - readSecret: jest.fn(), - async readFile(path) { - const content = ({ - 'my-secret': 'secret', - 'my-data.json': '{"a":{"b":{"c":42}}}', - 'my-data.yaml': 'some:\n yaml:\n key: 7', - 'my-data.yml': 'different: { key: hello }', - 'invalid.yaml': 'foo: [}', - } as { [key: string]: string })[path]; - - if (!content) { - throw new Error('File not found!'); - } - return content; - }, -}; - -describe('readSecret', () => { - it('should read file secrets', async () => { - await expect(readSecret({ file: 'my-secret' }, ctx)).resolves.toBe( - 'secret', - ); - await expect(readSecret({ file: 'no-secret' }, ctx)).rejects.toThrow( - 'File not found!', - ); - }); - - it('should read present env secrets', async () => { - await expect(readSecret({ env: 'SECRET' }, ctx)).resolves.toBe('my-secret'); - await expect(readSecret({ env: 'NO_SECRET' }, ctx)).resolves.toBe( - undefined, - ); - }); - - it('should include extra files', async () => { - // New format with path in fragment - await expect( - readSecret({ include: 'my-data.json#a.b.c' }, ctx), - ).resolves.toBe(42); - await expect( - readSecret({ include: 'my-data.json#a.b' }, ctx), - ).resolves.toEqual({ c: 42 }); - await expect( - readSecret({ include: 'my-data.yaml#some.yaml.key' }, ctx), - ).resolves.toBe(7); - await expect(readSecret({ include: 'my-data.yaml' }, ctx)).resolves.toEqual( - { - some: { yaml: { key: 7 } }, - }, - ); - await expect( - readSecret({ include: 'my-data.yaml#' }, ctx), - ).resolves.toEqual({ - some: { yaml: { key: 7 } }, - }); - await expect( - readSecret({ include: 'my-data.yml#different.key' }, ctx), - ).resolves.toBe('hello'); - await expect( - readSecret({ include: 'no-data.yml#different.key' }, ctx), - ).rejects.toThrow('File not found!'); - await expect( - readSecret({ include: 'my-data.yml#missing.key' }, ctx), - ).rejects.toThrow('Value is not an object at missing in my-data.yml'); - await expect(readSecret({ include: 'invalid.yaml' }, ctx)).rejects.toThrow( - 'Failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }', - ); - }); - - it('should reject invalid secrets', async () => { - await expect(readSecret('hello' as any, ctx)).rejects.toThrow( - 'secret must be a `object` type, but the final value was: `"hello"`.', - ); - await expect(readSecret({}, ctx)).rejects.toThrow( - "Secret must contain one of 'file', 'env', 'include'", - ); - await expect(readSecret({ unknown: 'derp' }, ctx)).rejects.toThrow( - "Secret must contain one of 'file', 'env', 'include'", - ); - await expect(readSecret({ include: 'no-parser.js' }, ctx)).rejects.toThrow( - 'No data secret parser available for extension .js', - ); - await expect( - readSecret({ include: 'my-data.yaml#some.wrong.yaml.key' }, ctx), - ).rejects.toThrow('Value is not an object at some.wrong in my-data.yaml'); - }); - - it('should have 100% test coverage', async () => { - let firstVisit = true; - const secret = {}; - const proto = { - get file() { - if (!firstVisit) { - Object.setPrototypeOf(secret, {}); - } - firstVisit = false; - return 'a-file'; - }, - }; - Object.setPrototypeOf(secret, proto); - - await expect(readSecret(secret, ctx)).rejects.toThrow( - 'Secret was left unhandled', - ); - }); -}); diff --git a/packages/config-loader/src/lib/secrets.ts b/packages/config-loader/src/lib/secrets.ts deleted file mode 100644 index 4ce87c6ace..0000000000 --- a/packages/config-loader/src/lib/secrets.ts +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as yup from 'yup'; -import yaml from 'yaml'; -import { extname } from 'path'; -import { JsonObject, JsonValue } from '@backstage/config'; -import { isObject, isNever } from './utils'; -import { ReaderContext } from './types'; - -// Reads a file and forwards the contents as is, assuming ut8 encoding -type FileSecret = { - // Path to the secret file, relative to the config file. - file: string; -}; - -// Reads the secret from an environment variable. -type EnvSecret = { - // The name of the environment file. - env: string; -}; - -// TODO(Rugvip): Move this out of secret reading when we remove the deprecated DataSecret and $secret format -type IncludeSecret = { - include: string; -}; - -type Secret = FileSecret | EnvSecret | IncludeSecret; - -// Schema for each type of secret description -const secretLoaderSchemas = { - file: yup.object({ - file: yup.string().required(), - }), - env: yup.object({ - env: yup.string().required(), - }), - include: yup.object({ - include: yup.string().required(), - }), -}; - -// The top-level secret schema, which figures out what type of secret it is. -const secretSchema = yup.lazy(value => { - if (typeof value !== 'object' || value === null) { - return yup.object().required().label('secret'); - } - - const loaderTypes = Object.keys( - secretLoaderSchemas, - ) as (keyof typeof secretLoaderSchemas)[]; - - for (const key of loaderTypes) { - if (key in value) { - return secretLoaderSchemas[key]; - } - } - throw new yup.ValidationError( - `Secret must contain one of '${loaderTypes.join("', '")}'`, - value, - '$secret', - ); -}); - -// Parsers for each type of data secret file. -const dataSecretParser: { - [ext in string]: (content: string) => Promise; -} = { - '.json': async content => JSON.parse(content), - '.yaml': async content => yaml.parse(content), - '.yml': async content => yaml.parse(content), -}; - -/** - * Transforms a secret description into the actual secret value. - */ -export async function readSecret( - data: JsonObject, - ctx: ReaderContext, -): Promise { - const secret = secretSchema.validateSync(data, { strict: true }) as Secret; - - if ('file' in secret) { - return ctx.readFile(secret.file); - } - if ('env' in secret) { - return ctx.env[secret.env]; - } - if ('include' in secret) { - const [filePath, dataPath] = secret.include.split(/#(.*)/); - - const ext = extname(filePath); - const parser = dataSecretParser[ext]; - if (!parser) { - throw new Error(`No data secret parser available for extension ${ext}`); - } - - const content = await ctx.readFile(filePath); - - const parts = dataPath ? dataPath.split('.') : []; - - let value: JsonValue | undefined; - try { - value = await parser(content); - } catch (error) { - throw new Error(`Failed to parse included file ${filePath}, ${error}`); - } - for (const [index, part] of parts.entries()) { - if (!isObject(value)) { - const errPath = parts.slice(0, index).join('.'); - throw new Error(`Value is not an object at ${errPath} in ${filePath}`); - } - value = value[part]; - } - - return value; - } - - isNever(); - throw new Error('Secret was left unhandled'); -} diff --git a/packages/config-loader/src/lib/transform.test.ts b/packages/config-loader/src/lib/transform.test.ts new file mode 100644 index 0000000000..633bae0a53 --- /dev/null +++ b/packages/config-loader/src/lib/transform.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { applyConfigTransforms } from './transform'; + +describe('applyConfigTransforms', () => { + it('should apply not transforms to input', async () => { + const data = applyConfigTransforms( + { + app: { + title: 'Test', + x: 1, + y: [null, true], + z: null, + }, + }, + [], + ); + + await expect(data).resolves.toEqual({ + app: { + title: 'Test', + x: 1, + y: [true], + }, + }); + }); + + it('should throw if input is not an object', async () => { + const config = applyConfigTransforms('not-config', []); + + await expect(config).rejects.toThrow('expected object at config root'); + }); + + it('should apply transforms', async () => { + const config = applyConfigTransforms( + { + app: { + title: 'Test', + x: 1, + y: [null, true], + z: null, + }, + }, + [ + async value => { + if (typeof value === 'number') { + return [true, value + 1]; + } + return [false, value]; + }, + async value => { + if (typeof value === 'string' && value.length > 1) { + return [true, value.split('')]; + } + return [false, value]; + }, + ], + ); + + await expect(config).resolves.toEqual({ + app: { + title: ['T', 'e', 's', 't'], + x: 2, + y: [true], + }, + }); + }); +}); diff --git a/packages/config-loader/src/lib/reader.ts b/packages/config-loader/src/lib/transform.ts similarity index 59% rename from packages/config-loader/src/lib/reader.ts rename to packages/config-loader/src/lib/transform.ts index 0996de6ce7..9edb6ae78b 100644 --- a/packages/config-loader/src/lib/reader.ts +++ b/packages/config-loader/src/lib/transform.ts @@ -14,29 +14,39 @@ * limitations under the License. */ -import { AppConfig, JsonObject, JsonValue } from '@backstage/config'; -import { basename } from 'path'; -import yaml from 'yaml'; -import { ReaderContext } from './types'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { TransformFunc } from './types'; import { isObject } from './utils'; /** * Reads and parses, and validates, and transforms a single config file. * The transformation rewrites any special values, like the $secret key. */ -export async function readConfigFile( - filePath: string, - ctx: ReaderContext, -): Promise { - const configYaml = await ctx.readFile(filePath); - const config = yaml.parse(configYaml); - - const context = basename(filePath); - +export async function applyConfigTransforms( + input: JsonValue, + transforms: TransformFunc[], +): Promise { async function transform( - obj: JsonValue, + inputObj: JsonValue, path: string, ): Promise { + let obj = inputObj; + + for (const tf of transforms) { + try { + const [applied, newObj] = await tf(inputObj); + if (applied) { + if (newObj === undefined) { + return newObj; + } + obj = newObj; + break; + } + } catch (error) { + throw new Error(`error at ${path}, ${error.message}`); + } + } + if (typeof obj !== 'object') { return obj; } else if (obj === null) { @@ -54,24 +64,6 @@ export async function readConfigFile( return arr; } - // Check if there's any key that starts with a '$', in that case we treat - // this entire object as a secret. - const [secretKey] = Object.keys(obj).filter(key => key.startsWith('$')); - if (secretKey) { - if (Object.keys(obj).length !== 1) { - throw new Error( - `Secret key '${secretKey}' has adjacent keys at ${path}`, - ); - } - try { - return await ctx.readSecret(path, { - [secretKey.slice(1)]: obj[secretKey], - }); - } catch (error) { - throw new Error(`Invalid secret at ${path}: ${error.message}`); - } - } - const out: JsonObject = {}; for (const [key, value] of Object.entries(obj)) { @@ -87,9 +79,9 @@ export async function readConfigFile( return out; } - const finalConfig = await transform(config, ''); - if (!isObject(finalConfig)) { - throw new TypeError('Expected object at config root'); + const finalData = await transform(input, ''); + if (!isObject(finalData)) { + throw new TypeError('expected object at config root'); } - return { data: finalConfig, context }; + return finalData; } diff --git a/packages/config-loader/src/lib/types.ts b/packages/config-loader/src/lib/types.ts index 95e1d6655e..b632547485 100644 --- a/packages/config-loader/src/lib/types.ts +++ b/packages/config-loader/src/lib/types.ts @@ -14,20 +14,12 @@ * limitations under the License. */ -import { JsonObject, JsonValue } from '@backstage/config'; +import { JsonValue } from '@backstage/config'; + +export type EnvFunc = (name: string) => Promise; export type ReadFileFunc = (path: string) => Promise; -export type ReadSecretFunc = ( - path: string, - desc: JsonObject, -) => Promise; -export type SkipFunc = (path: string) => boolean; -/** - * Common context that provides all the necessary hooks for reading configuration files. - */ -export type ReaderContext = { - env: { [name in string]?: string }; - readFile: ReadFileFunc; - readSecret: ReadSecretFunc; -}; +export type TransformFunc = ( + value: JsonValue, +) => Promise<[boolean, JsonValue | undefined]>; diff --git a/packages/config-loader/src/lib/utils.ts b/packages/config-loader/src/lib/utils.ts index 37145ec971..9a72bc3c0e 100644 --- a/packages/config-loader/src/lib/utils.ts +++ b/packages/config-loader/src/lib/utils.ts @@ -24,8 +24,3 @@ export function isObject(obj: JsonValue | undefined): obj is JsonObject { } return obj !== null; } - -// A thing to make sure we've narrowed the type down to never -export function isNever() { - return void 0 as T; -} diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 11cb1dc338..71553a3dd7 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -15,9 +15,14 @@ */ import fs from 'fs-extra'; -import { resolve as resolvePath, dirname, isAbsolute } from 'path'; -import { AppConfig, JsonObject, JsonValue } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret } from './lib'; +import yaml from 'yaml'; +import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { AppConfig } from '@backstage/config'; +import { + applyConfigTransforms, + readEnvConfig, + createIncludeTransform, +} from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -30,30 +35,6 @@ export type LoadConfigOptions = { env: string; }; -class Context { - constructor( - private readonly options: { - env: { [name in string]?: string }; - rootPath: string; - }, - ) {} - - get env() { - return this.options.env; - } - - async readFile(path: string): Promise { - return fs.readFile(resolvePath(this.options.rootPath, path), 'utf8'); - } - - async readSecret( - _path: string, - desc: JsonObject, - ): Promise { - return readSecret(desc, this); - } -} - export async function loadConfig( options: LoadConfigOptions, ): Promise { @@ -82,24 +63,28 @@ export async function loadConfig( } } + const env = async (name: string) => process.env[name]; + try { for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); } - const config = await readConfigFile( - configPath, - new Context({ - env: process.env, - rootPath: dirname(configPath), - }), - ); - configs.push(config); + const dir = dirname(configPath); + const readFile = (path: string) => + fs.readFile(resolvePath(dir, path), 'utf8'); + + const input = yaml.parse(await readFile(configPath)); + const data = await applyConfigTransforms(input, [ + createIncludeTransform(env, readFile), + ]); + + configs.push({ data, context: basename(configPath) }); } } catch (error) { throw new Error( - `Failed to read static configuration file: ${error.message}`, + `Failed to read static configuration file, ${error.message}`, ); }