packages/config-loader: split loader into lib

packages/config-loader/src/index.ts

@@ -15,4 +15,4 @@
  */
 
 export { loadConfig } from './loader';
-export type { LoadConfigOptions } from './types';
+export type { LoadConfigOptions } from './loader';

packages/config-loader/src/lib/env.test.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { readEnv } from './loader';
+import { readEnv } from './env';
 
 describe('readEnv', () => {
   it('should return empty config for empty env', () => {

packages/config-loader/src/lib/env.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AppConfig, JsonObject } from '@backstage/config';
+
+const ENV_PREFIX = 'APP_CONFIG_';
+
+// Update the same pattern in config package if this is changed
+const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
+
+export function readEnv(env: {
+  [name: string]: string | undefined;
+}): AppConfig[] {
+  let config: JsonObject | undefined = undefined;
+
+  for (const [name, value] of Object.entries(env)) {
+    if (!value) {
+      continue;
+    }
+    if (name.startsWith(ENV_PREFIX)) {
+      const key = name.replace(ENV_PREFIX, '');
+      const keyParts = key.split('_');
+
+      let obj = (config = config ?? {});
+      for (const [index, part] of keyParts.entries()) {
+        if (!CONFIG_KEY_PART_PATTERN.test(part)) {
+          throw new TypeError(`Invalid env config key '${key}'`);
+        }
+        if (index < keyParts.length - 1) {
+          obj = (obj[part] = obj[part] ?? {}) as JsonObject;
+          if (typeof obj !== 'object' || Array.isArray(obj)) {
+            const subKey = keyParts.slice(0, index + 1).join('_');
+            throw new TypeError(
+              `Could not nest config for key '${key}' under existing value '${subKey}'`,
+            );
+          }
+        } else {
+          if (part in obj) {
+            throw new TypeError(
+              `Refusing to override existing config at key '${key}'`,
+            );
+          }
+          try {
+            const parsedValue = JSON.parse(value);
+            if (parsedValue === null) {
+              throw new Error('value may not be null');
+            }
+            obj[part] = parsedValue;
+          } catch (error) {
+            throw new TypeError(
+              `Failed to parse JSON-serialized config value for key '${key}', ${error}`,
+            );
+          }
+        }
+      }
+    }
+  }
+
+  return config ? [config] : [];
+}

packages/config-loader/src/lib/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { findRootPath } from './paths';
+export { readConfigFile } from './reader';
+export { readEnv } from './env';

packages/config-loader/src/lib/reader.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yaml from 'yaml';
+import { isObject } from './utils';
+import { readSecret } from './secrets';
+import { JsonValue, JsonObject } from '@backstage/config';
+import { ReaderContext } from './types';
+
+export async function readConfigFile(filePath: string, ctx: ReaderContext) {
+  const configYaml = await ctx.readFile(filePath);
+  const config = yaml.parse(configYaml);
+
+  async function transform(
+    obj: JsonValue,
+    path: string,
+  ): Promise<JsonValue | undefined> {
+    if (typeof obj !== 'object') {
+      return obj;
+    } else if (obj === null) {
+      return obj;
+    } else if (Array.isArray(obj)) {
+      const arr = new Array<JsonValue>();
+
+      for (const [index, value] of obj.entries()) {
+        const out = await transform(value, `${path}[${index}]`);
+        if (out !== undefined) {
+          arr.push(out);
+        }
+      }
+
+      return arr;
+    }
+
+    if ('$secret' in obj) {
+      if (!isObject(obj.$secret)) {
+        throw TypeError(`Expected object at secret ${path}.$secret`);
+      }
+
+      try {
+        return await readSecret(obj.$secret, ctx);
+      } catch (error) {
+        throw new Error(`Invalid secret at ${path}: ${error.message}`);
+      }
+    }
+
+    const out: JsonObject = {};
+
+    for (const [key, value] of Object.entries(obj)) {
+      const result = await transform(value, `${path}.${key}`);
+      if (result !== undefined) {
+        out[key] = result;
+      }
+    }
+
+    return out;
+  }
+
+  const finalConfig = await transform(config, '');
+  if (!isObject(finalConfig)) {
+    throw new TypeError('Expected object at config root');
+  }
+  return finalConfig;
+}

packages/config-loader/src/lib/secrets.ts

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as yup from 'yup';
+import yaml from 'yaml';
+import { extname } from 'path';
+import { JsonObject, JsonValue } from '@backstage/config';
+import { isObject, isNever } from './utils';
+import { ReaderContext } from './types';
+
+type FileSecret = {
+  file: string;
+};
+
+type EnvSecret = {
+  env: string;
+};
+
+type DataSecret = {
+  data: string;
+  path: string | string[];
+};
+
+type Secret = FileSecret | EnvSecret | DataSecret;
+
+const secretLoaderSchemas = {
+  file: yup.object({
+    file: yup.string().required(),
+  }),
+  env: yup.object({
+    env: yup.string().required(),
+  }),
+  data: yup.object({
+    data: yup.string().required(),
+    path: yup.lazy(value => {
+      if (typeof value === 'string') {
+        return yup.string().required();
+      }
+      return yup.array().of(yup.string().required()).required();
+    }),
+  }),
+};
+
+const secretSchema = yup.lazy<object>(value => {
+  if (typeof value !== 'object' || value === null) {
+    return yup.object().required();
+  }
+
+  const loaderTypes = Object.keys(
+    secretLoaderSchemas,
+  ) as (keyof typeof secretLoaderSchemas)[];
+
+  for (const key of loaderTypes) {
+    if (key in value) {
+      return secretLoaderSchemas[key];
+    }
+  }
+  throw new yup.ValidationError(
+    `Secret must contain one of '${loaderTypes.join("', '")}'`,
+    value,
+    '$secret',
+  );
+});
+
+const dataSecretParser: {
+  [ext in string]: (content: string) => Promise<JsonObject>;
+} = {
+  '.json': async content => JSON.parse(content),
+  '.yaml': async content => yaml.parse(content),
+  '.yml': async content => yaml.parse(content),
+};
+
+export async function readSecret(
+  data: JsonObject,
+  ctx: ReaderContext,
+): Promise<string | undefined> {
+  if (!ctx.shouldReadSecrets) {
+    return undefined;
+  }
+
+  const secret = secretSchema.validateSync(data) as Secret;
+
+  if ('file' in secret) {
+    return ctx.readFile(secret.file);
+  }
+  if ('env' in secret) {
+    return ctx.env[secret.env];
+  }
+  if ('data' in secret) {
+    const ext = extname(secret.data);
+    const parser = dataSecretParser[ext];
+    if (!parser) {
+      throw new Error(`No data secret parser available for extension ${ext}`);
+    }
+
+    const content = await ctx.readFile(secret.data);
+
+    const { path } = secret;
+    const parts = typeof path === 'string' ? path.split('.') : path;
+
+    let value: JsonValue = await parser(content);
+    for (const [index, part] of parts.entries()) {
+      if (!isObject(value)) {
+        const errPath = parts.slice(0, index).join('.');
+        throw new Error(
+          `Value is not an object at ${errPath} in ${secret.data}`,
+        );
+      }
+      value = value[part];
+    }
+    return String(value);
+  }
+
+  isNever<typeof secret>();
+  throw new Error('Secret was left unhandled');
+}

packages/config-loader/src/lib/types.ts

@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-export type LoadConfigOptions = {
-  // Config path, defaults to app-config.yaml in project root
-  configPath?: string;
+export type ReadFileFunc = (path: string) => Promise<string>;
 
-  // Whether to read secrets or omit them, defaults to false.
-  shouldReadSecrets?: boolean;
+export type ReaderContext = {
+  shouldReadSecrets: boolean;
+  readFile: ReadFileFunc;
+  env: { [name in string]?: string };
 };

packages/config-loader/src/lib/utils.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { JsonValue, JsonObject } from '@backstage/config';
+
+export function isObject(obj: JsonValue | undefined): obj is JsonObject {
+  if (typeof obj !== 'object') {
+    return false;
+  } else if (Array.isArray(obj)) {
+    return false;
+  }
+  return obj !== null;
+}
+
+// A thing to make sure we've narrowed the type down to never
+export function isNever<T extends never>() {
+  return void 0 as T;
+}

packages/config-loader/src/loader.ts

@@ -15,254 +15,18 @@
  */
 
 import fs from 'fs-extra';
-import * as yup from 'yup';
-import yaml from 'yaml';
-import { resolve as resolvePath, dirname, extname } from 'path';
-import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
-import { findRootPath } from './paths';
-import { LoadConfigOptions } from './types';
+import { resolve as resolvePath, dirname } from 'path';
+import { AppConfig } from '@backstage/config';
+import { findRootPath, readConfigFile, readEnv } from './lib';
 
-const ENV_PREFIX = 'APP_CONFIG_';
+export type LoadConfigOptions = {
+  // Config path, defaults to app-config.yaml in project root
+  configPath?: string;
 
-// Update the same pattern in config package if this is changed
-const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
-
-export function readEnv(env: {
-  [name: string]: string | undefined;
-}): AppConfig[] {
-  let config: JsonObject | undefined = undefined;
-
-  for (const [name, value] of Object.entries(env)) {
-    if (!value) {
-      continue;
-    }
-    if (name.startsWith(ENV_PREFIX)) {
-      const key = name.replace(ENV_PREFIX, '');
-      const keyParts = key.split('_');
-
-      let obj = (config = config ?? {});
-      for (const [index, part] of keyParts.entries()) {
-        if (!CONFIG_KEY_PART_PATTERN.test(part)) {
-          throw new TypeError(`Invalid env config key '${key}'`);
-        }
-        if (index < keyParts.length - 1) {
-          obj = (obj[part] = obj[part] ?? {}) as JsonObject;
-          if (typeof obj !== 'object' || Array.isArray(obj)) {
-            const subKey = keyParts.slice(0, index + 1).join('_');
-            throw new TypeError(
-              `Could not nest config for key '${key}' under existing value '${subKey}'`,
-            );
-          }
-        } else {
-          if (part in obj) {
-            throw new TypeError(
-              `Refusing to override existing config at key '${key}'`,
-            );
-          }
-          try {
-            const parsedValue = JSON.parse(value);
-            if (parsedValue === null) {
-              throw new Error('value may not be null');
-            }
-            obj[part] = parsedValue;
-          } catch (error) {
-            throw new TypeError(
-              `Failed to parse JSON-serialized config value for key '${key}', ${error}`,
-            );
-          }
-        }
-      }
-    }
-  }
-
-  return config ? [config] : [];
-}
-
-type ReadFileFunc = (path: string) => Promise<string>;
-
-type ReaderContext = {
-  shouldReadSecrets: boolean;
-  readFile: ReadFileFunc;
-  env: { [name in string]?: string };
+  // Whether to read secrets or omit them, defaults to false.
+  shouldReadSecrets?: boolean;
 };
 
-function isObject(obj: JsonValue | undefined): obj is JsonObject {
-  if (typeof obj !== 'object') {
-    return false;
-  } else if (Array.isArray(obj)) {
-    return false;
-  }
-  return obj !== null;
-}
-
-type FileSecret = {
-  file: string;
-};
-
-type EnvSecret = {
-  env: string;
-};
-
-type DataSecret = {
-  data: string;
-  path: string | string[];
-};
-
-type Secret = FileSecret | EnvSecret | DataSecret;
-
-const secretLoaderSchemas = {
-  file: yup.object({
-    file: yup.string().required(),
-  }),
-  env: yup.object({
-    env: yup.string().required(),
-  }),
-  data: yup.object({
-    data: yup.string().required(),
-    path: yup.lazy(value => {
-      if (typeof value === 'string') {
-        return yup.string().required();
-      }
-      return yup.array().of(yup.string().required()).required();
-    }),
-  }),
-};
-
-const secretSchema = yup.lazy<object>(value => {
-  if (typeof value !== 'object' || value === null) {
-    return yup.object().required();
-  }
-
-  const loaderTypes = Object.keys(
-    secretLoaderSchemas,
-  ) as (keyof typeof secretLoaderSchemas)[];
-
-  for (const key of loaderTypes) {
-    if (key in value) {
-      return secretLoaderSchemas[key];
-    }
-  }
-  throw new yup.ValidationError(
-    `Secret must contain one of '${loaderTypes.join("', '")}'`,
-    value,
-    '$secret',
-  );
-});
-
-const dataSecretParser: {
-  [ext in string]: (content: string) => Promise<JsonObject>;
-} = {
-  '.json': async content => JSON.parse(content),
-  '.yaml': async content => yaml.parse(content),
-  '.yml': async content => yaml.parse(content),
-};
-
-// A thing to make sure we've narrowed the type down to never
-function isNever<T extends never>() {
-  return void 0 as T;
-}
-
-export async function readSecret(
-  data: JsonObject,
-  ctx: ReaderContext,
-): Promise<string | undefined> {
-  if (!ctx.shouldReadSecrets) {
-    return undefined;
-  }
-
-  const secret = secretSchema.validateSync(data) as Secret;
-
-  if ('file' in secret) {
-    return ctx.readFile(secret.file);
-  }
-  if ('env' in secret) {
-    return ctx.env[secret.env];
-  }
-  if ('data' in secret) {
-    const ext = extname(secret.data);
-    const parser = dataSecretParser[ext];
-    if (!parser) {
-      throw new Error(`No data secret parser available for extension ${ext}`);
-    }
-
-    const content = await ctx.readFile(secret.data);
-
-    const { path } = secret;
-    const parts = typeof path === 'string' ? path.split('.') : path;
-
-    let value: JsonValue = await parser(content);
-    for (const [index, part] of parts.entries()) {
-      if (!isObject(value)) {
-        const errPath = parts.slice(0, index).join('.');
-        throw new Error(
-          `Value is not an object at ${errPath} in ${secret.data}`,
-        );
-      }
-      value = value[part];
-    }
-    return String(value);
-  }
-
-  isNever<typeof secret>();
-  throw new Error('Secret was left unhandled');
-}
-
-export async function readConfigFile(filePath: string, ctx: ReaderContext) {
-  const configYaml = await ctx.readFile(filePath);
-  const config = yaml.parse(configYaml);
-
-  async function transform(
-    obj: JsonValue,
-    path: string,
-  ): Promise<JsonValue | undefined> {
-    if (typeof obj !== 'object') {
-      return obj;
-    } else if (obj === null) {
-      return obj;
-    } else if (Array.isArray(obj)) {
-      const arr = new Array<JsonValue>();
-
-      for (const [index, value] of obj.entries()) {
-        const out = await transform(value, `${path}[${index}]`);
-        if (out !== undefined) {
-          arr.push(out);
-        }
-      }
-
-      return arr;
-    }
-
-    if ('$secret' in obj) {
-      if (!isObject(obj.$secret)) {
-        throw TypeError(`Expected object at secret ${path}.$secret`);
-      }
-
-      try {
-        return await readSecret(obj.$secret, ctx);
-      } catch (error) {
-        throw new Error(`Invalid secret at ${path}: ${error.message}`);
-      }
-    }
-
-    const out: JsonObject = {};
-
-    for (const [key, value] of Object.entries(obj)) {
-      const result = await transform(value, `${path}.${key}`);
-      if (result !== undefined) {
-        out[key] = result;
-      }
-    }
-
-    return out;
-  }
-
-  const finalConfig = await transform(config, '');
-  if (!isObject(finalConfig)) {
-    throw new TypeError('Expected object at config root');
-  }
-  return finalConfig;
-}
-
 export async function loadStaticConfig(
   options: LoadConfigOptions,
 ): Promise<AppConfig[]> {