diff --git a/packages/config-loader/src/lib/transform/apply.test.ts b/packages/config-loader/src/lib/transform/apply.test.ts index ab79f3805a..ea87734036 100644 --- a/packages/config-loader/src/lib/transform/apply.test.ts +++ b/packages/config-loader/src/lib/transform/apply.test.ts @@ -19,7 +19,6 @@ import { applyConfigTransforms } from './apply'; describe('applyConfigTransforms', () => { it('should apply not transforms to input', async () => { const data = applyConfigTransforms( - '', { app: { title: 'Test', @@ -28,6 +27,7 @@ describe('applyConfigTransforms', () => { z: null, }, }, + {}, [], ); @@ -41,14 +41,13 @@ describe('applyConfigTransforms', () => { }); it('should throw if input is not an object', async () => { - const config = applyConfigTransforms('', 'not-config', []); + const config = applyConfigTransforms('not-config', {}, []); await expect(config).rejects.toThrow('expected object at config root'); }); it('should apply transforms', async () => { const config = applyConfigTransforms( - '', { app: { title: 'Test', @@ -57,6 +56,7 @@ describe('applyConfigTransforms', () => { z: null, }, }, + {}, [ async value => { if (typeof value === 'number') { diff --git a/packages/config-loader/src/lib/transform/apply.ts b/packages/config-loader/src/lib/transform/apply.ts index 80ddad90d0..2e5c71f335 100644 --- a/packages/config-loader/src/lib/transform/apply.ts +++ b/packages/config-loader/src/lib/transform/apply.ts @@ -16,34 +16,36 @@ import { JsonObject, JsonValue } from '@backstage/types'; import { assertError } from '@backstage/errors'; -import { TransformFunc } from './types'; +import { EnvFunc, TransformContext, TransformFunc } from './types'; import { isObject } from './utils'; +import { createSubstitutionTransform } from './substitution'; +import { createIncludeTransform } from './include'; /** * Applies a set of transforms to raw configuration data. */ export async function applyConfigTransforms( - initialDir: string, input: JsonValue, + context: { dir?: string }, transforms: TransformFunc[], ): Promise { async function transform( inputObj: JsonValue, path: string, - baseDir: string, + baseDir?: string, ): Promise { let obj = inputObj; let dir = baseDir; for (const tf of transforms) { try { - const result = await tf(inputObj, baseDir); + const result = await tf(inputObj, { dir }); if (result.applied) { if (result.value === undefined) { return undefined; } obj = result.value; - dir = result.newBaseDir ?? dir; + dir = result?.newDir ?? dir; break; } } catch (error) { @@ -84,9 +86,36 @@ export async function applyConfigTransforms( return out; } - const finalData = await transform(input, '', initialDir); + const finalData = await transform(input, '', context?.dir); if (!isObject(finalData)) { throw new TypeError('expected object at config root'); } return finalData; } + +/** @internal */ +export type ConfigTransformer = ( + input: JsonObject, + context?: TransformContext, +) => Promise; + +/** @internal */ +export function createConfigTransformer(options: { + envFunc?: EnvFunc; + readFile?(path: string): Promise; +}): ConfigTransformer { + const { envFunc = async name => process.env[name], readFile } = options; + const substitutionTransform = createSubstitutionTransform(envFunc); + const transforms = [substitutionTransform]; + if (readFile) { + const includeTransform = createIncludeTransform( + envFunc, + readFile, + substitutionTransform, + ); + transforms.push(includeTransform); + } + + return async (input, ctx) => + applyConfigTransforms(input, ctx ?? {}, transforms); +} diff --git a/packages/config-loader/src/lib/transform/include.test.ts b/packages/config-loader/src/lib/transform/include.test.ts index c80efe6785..13dc3c407e 100644 --- a/packages/config-loader/src/lib/transform/include.test.ts +++ b/packages/config-loader/src/lib/transform/include.test.ts @@ -69,34 +69,34 @@ const includeTransform = createIncludeTransform(env, readFile, substitute); describe('includeTransform', () => { it('should not transform unknown values', async () => { - await expect(includeTransform('foo', root)).resolves.toEqual({ + await expect(includeTransform('foo', { dir: root })).resolves.toEqual({ applied: false, }); - await expect(includeTransform([1], root)).resolves.toEqual({ + await expect(includeTransform([1], { dir: root })).resolves.toEqual({ applied: false, }); - await expect(includeTransform(1, root)).resolves.toEqual({ + await expect(includeTransform(1, { dir: root })).resolves.toEqual({ applied: false, }); - await expect(includeTransform({ x: 'y' }, root)).resolves.toEqual({ + await expect(includeTransform({ x: 'y' }, { dir: root })).resolves.toEqual({ applied: false, }); - await expect(includeTransform(null, root)).resolves.toEqual({ + await expect(includeTransform(null, { dir: root })).resolves.toEqual({ applied: false, }); }); it('should include text files', async () => { await expect( - includeTransform({ $file: 'my-secret' }, root), + includeTransform({ $file: 'my-secret' }, { dir: root }), ).resolves.toEqual({ applied: true, value: 'secret' }); await expect( - includeTransform({ $file: 'no-secret' }, root), + includeTransform({ $file: 'no-secret' }, { dir: root }), ).rejects.toThrow('File not found!'); }); it('should trim newlines from end of file', async () => { await expect( - includeTransform({ $file: 'with-newline-at-the-end' }, root), + includeTransform({ $file: 'with-newline-at-the-end' }, { dir: root }), ).resolves.toEqual({ applied: true, value: 'value without newline at the end', @@ -104,12 +104,14 @@ describe('includeTransform', () => { }); it('should include env vars', async () => { - await expect(includeTransform({ $env: 'SECRET' }, root)).resolves.toEqual({ + await expect( + includeTransform({ $env: 'SECRET' }, { dir: root }), + ).resolves.toEqual({ applied: true, value: 'my-secret', }); await expect( - includeTransform({ $env: 'NO_SECRET' }, root), + includeTransform({ $env: 'NO_SECRET' }, { dir: root }), ).resolves.toEqual({ applied: true, value: undefined, @@ -119,16 +121,19 @@ describe('includeTransform', () => { it('should include config files', async () => { // New format with path in fragment await expect( - includeTransform({ $include: 'my-data.json#a.b.c' }, root), + includeTransform({ $include: 'my-data.json#a.b.c' }, { dir: root }), ).resolves.toEqual({ applied: true, value: 42 }); await expect( - includeTransform({ $include: 'my-data.json#a.b' }, root), + includeTransform({ $include: 'my-data.json#a.b' }, { dir: root }), ).resolves.toEqual({ applied: true, value: { c: 42 } }); await expect( - includeTransform({ $include: 'my-data.yaml#some.yaml.key' }, root), + includeTransform( + { $include: 'my-data.yaml#some.yaml.key' }, + { dir: root }, + ), ).resolves.toEqual({ applied: true, value: 7 }); await expect( - includeTransform({ $include: 'my-data.yaml' }, root), + includeTransform({ $include: 'my-data.yaml' }, { dir: root }), ).resolves.toEqual({ applied: true, value: { @@ -136,7 +141,7 @@ describe('includeTransform', () => { }, }); await expect( - includeTransform({ $include: 'my-data.yaml#' }, root), + includeTransform({ $include: 'my-data.yaml#' }, { dir: root }), ).resolves.toEqual({ applied: true, value: { @@ -144,26 +149,32 @@ describe('includeTransform', () => { }, }); await expect( - includeTransform({ $include: 'my-data.yml#different.key' }, root), + includeTransform( + { $include: 'my-data.yml#different.key' }, + { dir: root }, + ), ).resolves.toEqual({ applied: true, value: 'hello' }); }); it('should reject invalid includes', async () => { await expect( - includeTransform({ $include: 'no-parser.js' }, root), + includeTransform({ $include: 'no-parser.js' }, { dir: root }), ).rejects.toThrow( 'no configuration parser available for included file no-parser.js', ); await expect( - includeTransform({ $include: 'no-data.yml#different.key' }, root), + includeTransform( + { $include: 'no-data.yml#different.key' }, + { dir: root }, + ), ).rejects.toThrow('File not found!'); await expect( - includeTransform({ $include: 'my-data.yml#missing.key' }, root), + includeTransform({ $include: 'my-data.yml#missing.key' }, { dir: root }), ).rejects.toThrow( "value at 'missing' in included file my-data.yml is not an object", ); await expect( - includeTransform({ $include: 'invalid.yaml' }, root), + includeTransform({ $include: 'invalid.yaml' }, { dir: root }), ).rejects.toThrow( /failed to parse included file invalid.yaml, YAMLParseError: Flow sequence in block collection must be sufficiently indented and end with a \] at line 1, column 7:\s+foo: \[\}/, ); @@ -171,11 +182,14 @@ describe('includeTransform', () => { it('should call substitute prior to handling includes directive', async () => { await expect( - includeTransform({ $include: `${substituteMe}/my-data.json` }, root), + includeTransform( + { $include: `${substituteMe}/my-data.json` }, + { dir: root }, + ), ).resolves.toEqual({ applied: true, value: { foo: 'bar' }, - newBaseDir: resolvePath(root, mySubstitution), + newDir: resolvePath(root, mySubstitution), }); }); }); diff --git a/packages/config-loader/src/lib/transform/include.ts b/packages/config-loader/src/lib/transform/include.ts index 84e4a9294d..4b1f5a7ecf 100644 --- a/packages/config-loader/src/lib/transform/include.ts +++ b/packages/config-loader/src/lib/transform/include.ts @@ -37,7 +37,11 @@ export function createIncludeTransform( readFile: ReadFileFunc, substitute: TransformFunc, ): TransformFunc { - return async (input: JsonValue, baseDir: string) => { + return async (input, context) => { + const { dir } = context; + if (!dir) { + throw new Error('Include transform requires a base directory'); + } if (!isObject(input)) { return { applied: false }; } @@ -59,7 +63,7 @@ export function createIncludeTransform( throw new Error(`${includeKey} include value is not a string`); } - const substituteResults = await substitute(rawIncludedValue, baseDir); + const substituteResults = await substitute(rawIncludedValue, { dir }); const includeValue = substituteResults.applied ? substituteResults.value : rawIncludedValue; @@ -72,7 +76,7 @@ export function createIncludeTransform( switch (includeKey) { case '$file': try { - const value = await readFile(resolvePath(baseDir, includeValue)); + const value = await readFile(resolvePath(dir, includeValue)); return { applied: true, value: value.trimEnd() }; } catch (error) { throw new Error(`failed to read file ${includeValue}, ${error}`); @@ -95,9 +99,9 @@ export function createIncludeTransform( ); } - const path = resolvePath(baseDir, filePath); + const path = resolvePath(dir, filePath); const content = await readFile(path); - const newBaseDir = dirname(path); + const newDir = dirname(path); const parts = dataPath ? dataPath.split('.') : []; @@ -124,7 +128,7 @@ export function createIncludeTransform( return { applied: true, value, - newBaseDir: newBaseDir !== baseDir ? newBaseDir : undefined, + newDir: newDir !== dir ? newDir : undefined, }; } diff --git a/packages/config-loader/src/lib/transform/types.ts b/packages/config-loader/src/lib/transform/types.ts index 0b743cb0fe..9e7471ba75 100644 --- a/packages/config-loader/src/lib/transform/types.ts +++ b/packages/config-loader/src/lib/transform/types.ts @@ -20,9 +20,13 @@ export type EnvFunc = (name: string) => Promise; export type ReadFileFunc = (path: string) => Promise; +export interface TransformContext { + dir?: string; +} + export type TransformFunc = ( value: JsonValue, - baseDir: string, + context: TransformContext, ) => Promise< | { applied: false; @@ -30,6 +34,6 @@ export type TransformFunc = ( | { applied: true; value: JsonValue | undefined; - newBaseDir?: string | undefined; + newDir?: string | undefined; } >;