config-loader: transform refactor + high level helper

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/config-loader/src/lib/transform/apply.test.ts

@@ -19,7 +19,6 @@ import { applyConfigTransforms } from './apply';
 describe('applyConfigTransforms', () => {
   it('should apply not transforms to input', async () => {
     const data = applyConfigTransforms(
-      '',
       {
         app: {
           title: 'Test',
@@ -28,6 +27,7 @@ describe('applyConfigTransforms', () => {
           z: null,
         },
       },
+      {},
       [],
     );
 
@@ -41,14 +41,13 @@ describe('applyConfigTransforms', () => {
   });
 
   it('should throw if  input is not an object', async () => {
-    const config = applyConfigTransforms('', 'not-config', []);
+    const config = applyConfigTransforms('not-config', {}, []);
 
     await expect(config).rejects.toThrow('expected object at config root');
   });
 
   it('should apply transforms', async () => {
     const config = applyConfigTransforms(
-      '',
       {
         app: {
           title: 'Test',
@@ -57,6 +56,7 @@ describe('applyConfigTransforms', () => {
           z: null,
         },
       },
+      {},
       [
         async value => {
           if (typeof value === 'number') {

packages/config-loader/src/lib/transform/apply.ts

@@ -16,34 +16,36 @@
 
 import { JsonObject, JsonValue } from '@backstage/types';
 import { assertError } from '@backstage/errors';
-import { TransformFunc } from './types';
+import { EnvFunc, TransformContext, TransformFunc } from './types';
 import { isObject } from './utils';
+import { createSubstitutionTransform } from './substitution';
+import { createIncludeTransform } from './include';
 
 /**
  * Applies a set of transforms to raw configuration data.
  */
 export async function applyConfigTransforms(
-  initialDir: string,
   input: JsonValue,
+  context: { dir?: string },
   transforms: TransformFunc[],
 ): Promise<JsonObject> {
   async function transform(
     inputObj: JsonValue,
     path: string,
-    baseDir: string,
+    baseDir?: string,
   ): Promise<JsonValue | undefined> {
     let obj = inputObj;
     let dir = baseDir;
 
     for (const tf of transforms) {
       try {
-        const result = await tf(inputObj, baseDir);
+        const result = await tf(inputObj, { dir });
         if (result.applied) {
           if (result.value === undefined) {
             return undefined;
           }
           obj = result.value;
-          dir = result.newBaseDir ?? dir;
+          dir = result?.newDir ?? dir;
           break;
         }
       } catch (error) {
@@ -84,9 +86,36 @@ export async function applyConfigTransforms(
     return out;
   }
 
-  const finalData = await transform(input, '', initialDir);
+  const finalData = await transform(input, '', context?.dir);
   if (!isObject(finalData)) {
     throw new TypeError('expected object at config root');
   }
   return finalData;
 }
+
+/** @internal */
+export type ConfigTransformer = (
+  input: JsonObject,
+  context?: TransformContext,
+) => Promise<JsonObject>;
+
+/** @internal */
+export function createConfigTransformer(options: {
+  envFunc?: EnvFunc;
+  readFile?(path: string): Promise<string>;
+}): ConfigTransformer {
+  const { envFunc = async name => process.env[name], readFile } = options;
+  const substitutionTransform = createSubstitutionTransform(envFunc);
+  const transforms = [substitutionTransform];
+  if (readFile) {
+    const includeTransform = createIncludeTransform(
+      envFunc,
+      readFile,
+      substitutionTransform,
+    );
+    transforms.push(includeTransform);
+  }
+
+  return async (input, ctx) =>
+    applyConfigTransforms(input, ctx ?? {}, transforms);
+}

packages/config-loader/src/lib/transform/include.test.ts

@@ -69,34 +69,34 @@ const includeTransform = createIncludeTransform(env, readFile, substitute);
 
 describe('includeTransform', () => {
   it('should not transform unknown values', async () => {
-    await expect(includeTransform('foo', root)).resolves.toEqual({
+    await expect(includeTransform('foo', { dir: root })).resolves.toEqual({
       applied: false,
     });
-    await expect(includeTransform([1], root)).resolves.toEqual({
+    await expect(includeTransform([1], { dir: root })).resolves.toEqual({
       applied: false,
     });
-    await expect(includeTransform(1, root)).resolves.toEqual({
+    await expect(includeTransform(1, { dir: root })).resolves.toEqual({
       applied: false,
     });
-    await expect(includeTransform({ x: 'y' }, root)).resolves.toEqual({
+    await expect(includeTransform({ x: 'y' }, { dir: root })).resolves.toEqual({
       applied: false,
     });
-    await expect(includeTransform(null, root)).resolves.toEqual({
+    await expect(includeTransform(null, { dir: root })).resolves.toEqual({
       applied: false,
     });
   });
 
   it('should include text files', async () => {
     await expect(
-      includeTransform({ $file: 'my-secret' }, root),
+      includeTransform({ $file: 'my-secret' }, { dir: root }),
     ).resolves.toEqual({ applied: true, value: 'secret' });
     await expect(
-      includeTransform({ $file: 'no-secret' }, root),
+      includeTransform({ $file: 'no-secret' }, { dir: root }),
     ).rejects.toThrow('File not found!');
   });
   it('should trim newlines from end of file', async () => {
     await expect(
-      includeTransform({ $file: 'with-newline-at-the-end' }, root),
+      includeTransform({ $file: 'with-newline-at-the-end' }, { dir: root }),
     ).resolves.toEqual({
       applied: true,
       value: 'value without newline at the end',
@@ -104,12 +104,14 @@ describe('includeTransform', () => {
   });
 
   it('should include env vars', async () => {
-    await expect(includeTransform({ $env: 'SECRET' }, root)).resolves.toEqual({
+    await expect(
+      includeTransform({ $env: 'SECRET' }, { dir: root }),
+    ).resolves.toEqual({
       applied: true,
       value: 'my-secret',
     });
     await expect(
-      includeTransform({ $env: 'NO_SECRET' }, root),
+      includeTransform({ $env: 'NO_SECRET' }, { dir: root }),
     ).resolves.toEqual({
       applied: true,
       value: undefined,
@@ -119,16 +121,19 @@ describe('includeTransform', () => {
   it('should include config files', async () => {
     // New format with path in fragment
     await expect(
-      includeTransform({ $include: 'my-data.json#a.b.c' }, root),
+      includeTransform({ $include: 'my-data.json#a.b.c' }, { dir: root }),
     ).resolves.toEqual({ applied: true, value: 42 });
     await expect(
-      includeTransform({ $include: 'my-data.json#a.b' }, root),
+      includeTransform({ $include: 'my-data.json#a.b' }, { dir: root }),
     ).resolves.toEqual({ applied: true, value: { c: 42 } });
     await expect(
-      includeTransform({ $include: 'my-data.yaml#some.yaml.key' }, root),
+      includeTransform(
+        { $include: 'my-data.yaml#some.yaml.key' },
+        { dir: root },
+      ),
     ).resolves.toEqual({ applied: true, value: 7 });
     await expect(
-      includeTransform({ $include: 'my-data.yaml' }, root),
+      includeTransform({ $include: 'my-data.yaml' }, { dir: root }),
     ).resolves.toEqual({
       applied: true,
       value: {
@@ -136,7 +141,7 @@ describe('includeTransform', () => {
       },
     });
     await expect(
-      includeTransform({ $include: 'my-data.yaml#' }, root),
+      includeTransform({ $include: 'my-data.yaml#' }, { dir: root }),
     ).resolves.toEqual({
       applied: true,
       value: {
@@ -144,26 +149,32 @@ describe('includeTransform', () => {
       },
     });
     await expect(
-      includeTransform({ $include: 'my-data.yml#different.key' }, root),
+      includeTransform(
+        { $include: 'my-data.yml#different.key' },
+        { dir: root },
+      ),
     ).resolves.toEqual({ applied: true, value: 'hello' });
   });
 
   it('should reject invalid includes', async () => {
     await expect(
-      includeTransform({ $include: 'no-parser.js' }, root),
+      includeTransform({ $include: 'no-parser.js' }, { dir: root }),
     ).rejects.toThrow(
       'no configuration parser available for included file no-parser.js',
     );
     await expect(
-      includeTransform({ $include: 'no-data.yml#different.key' }, root),
+      includeTransform(
+        { $include: 'no-data.yml#different.key' },
+        { dir: root },
+      ),
     ).rejects.toThrow('File not found!');
     await expect(
-      includeTransform({ $include: 'my-data.yml#missing.key' }, root),
+      includeTransform({ $include: 'my-data.yml#missing.key' }, { dir: root }),
     ).rejects.toThrow(
       "value at 'missing' in included file my-data.yml is not an object",
     );
     await expect(
-      includeTransform({ $include: 'invalid.yaml' }, root),
+      includeTransform({ $include: 'invalid.yaml' }, { dir: root }),
     ).rejects.toThrow(
       /failed to parse included file invalid.yaml, YAMLParseError: Flow sequence in block collection must be sufficiently indented and end with a \] at line 1, column 7:\s+foo: \[\}/,
     );
@@ -171,11 +182,14 @@ describe('includeTransform', () => {
 
   it('should call substitute prior to handling includes directive', async () => {
     await expect(
-      includeTransform({ $include: `${substituteMe}/my-data.json` }, root),
+      includeTransform(
+        { $include: `${substituteMe}/my-data.json` },
+        { dir: root },
+      ),
     ).resolves.toEqual({
       applied: true,
       value: { foo: 'bar' },
-      newBaseDir: resolvePath(root, mySubstitution),
+      newDir: resolvePath(root, mySubstitution),
     });
   });
 });

packages/config-loader/src/lib/transform/include.ts

@@ -37,7 +37,11 @@ export function createIncludeTransform(
   readFile: ReadFileFunc,
   substitute: TransformFunc,
 ): TransformFunc {
-  return async (input: JsonValue, baseDir: string) => {
+  return async (input, context) => {
+    const { dir } = context;
+    if (!dir) {
+      throw new Error('Include transform requires a base directory');
+    }
     if (!isObject(input)) {
       return { applied: false };
     }
@@ -59,7 +63,7 @@ export function createIncludeTransform(
       throw new Error(`${includeKey} include value is not a string`);
     }
 
-    const substituteResults = await substitute(rawIncludedValue, baseDir);
+    const substituteResults = await substitute(rawIncludedValue, { dir });
     const includeValue = substituteResults.applied
       ? substituteResults.value
       : rawIncludedValue;
@@ -72,7 +76,7 @@ export function createIncludeTransform(
     switch (includeKey) {
       case '$file':
         try {
-          const value = await readFile(resolvePath(baseDir, includeValue));
+          const value = await readFile(resolvePath(dir, includeValue));
           return { applied: true, value: value.trimEnd() };
         } catch (error) {
           throw new Error(`failed to read file ${includeValue}, ${error}`);
@@ -95,9 +99,9 @@ export function createIncludeTransform(
           );
         }
 
-        const path = resolvePath(baseDir, filePath);
+        const path = resolvePath(dir, filePath);
         const content = await readFile(path);
-        const newBaseDir = dirname(path);
+        const newDir = dirname(path);
 
         const parts = dataPath ? dataPath.split('.') : [];
 
@@ -124,7 +128,7 @@ export function createIncludeTransform(
         return {
           applied: true,
           value,
-          newBaseDir: newBaseDir !== baseDir ? newBaseDir : undefined,
+          newDir: newDir !== dir ? newDir : undefined,
         };
       }
 

packages/config-loader/src/lib/transform/types.ts

@@ -20,9 +20,13 @@ export type EnvFunc = (name: string) => Promise<string | undefined>;
 
 export type ReadFileFunc = (path: string) => Promise<string>;
 
+export interface TransformContext {
+  dir?: string;
+}
+
 export type TransformFunc = (
   value: JsonValue,
-  baseDir: string,
+  context: TransformContext,
 ) => Promise<
   | {
       applied: false;
@@ -30,6 +34,6 @@ export type TransformFunc = (
   | {
       applied: true;
       value: JsonValue | undefined;
-      newBaseDir?: string | undefined;
+      newDir?: string | undefined;
     }
 >;