From 3a2c353ad7c953504e50022b77254db8435a7953 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 12 Jun 2020 17:27:53 +0200 Subject: [PATCH] packages/config-loader: split loader into lib --- packages/config-loader/src/index.ts | 2 +- .../src/{loader.test.ts => lib/env.test.ts} | 2 +- packages/config-loader/src/lib/env.ts | 73 +++++ packages/config-loader/src/lib/index.ts | 19 ++ .../config-loader/src/{ => lib}/paths.test.ts | 0 packages/config-loader/src/{ => lib}/paths.ts | 0 packages/config-loader/src/lib/reader.ts | 77 ++++++ packages/config-loader/src/lib/secrets.ts | 129 +++++++++ packages/config-loader/src/{ => lib}/types.ts | 10 +- packages/config-loader/src/lib/utils.ts | 31 +++ packages/config-loader/src/loader.ts | 252 +----------------- 11 files changed, 344 insertions(+), 251 deletions(-) rename packages/config-loader/src/{loader.test.ts => lib/env.test.ts} (98%) create mode 100644 packages/config-loader/src/lib/env.ts create mode 100644 packages/config-loader/src/lib/index.ts rename packages/config-loader/src/{ => lib}/paths.test.ts (100%) rename packages/config-loader/src/{ => lib}/paths.ts (100%) create mode 100644 packages/config-loader/src/lib/reader.ts create mode 100644 packages/config-loader/src/lib/secrets.ts rename packages/config-loader/src/{ => lib}/types.ts (73%) create mode 100644 packages/config-loader/src/lib/utils.ts diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 822fb53d04..6738611a17 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,4 +15,4 @@ */ export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './types'; +export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/lib/env.test.ts similarity index 98% rename from packages/config-loader/src/loader.test.ts rename to packages/config-loader/src/lib/env.test.ts index 4c7a120c63..6a0a115b24 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/lib/env.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { readEnv } from './loader'; +import { readEnv } from './env'; describe('readEnv', () => { it('should return empty config for empty env', () => { diff --git a/packages/config-loader/src/lib/env.ts b/packages/config-loader/src/lib/env.ts new file mode 100644 index 0000000000..e9087c4387 --- /dev/null +++ b/packages/config-loader/src/lib/env.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig, JsonObject } from '@backstage/config'; + +const ENV_PREFIX = 'APP_CONFIG_'; + +// Update the same pattern in config package if this is changed +const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i; + +export function readEnv(env: { + [name: string]: string | undefined; +}): AppConfig[] { + let config: JsonObject | undefined = undefined; + + for (const [name, value] of Object.entries(env)) { + if (!value) { + continue; + } + if (name.startsWith(ENV_PREFIX)) { + const key = name.replace(ENV_PREFIX, ''); + const keyParts = key.split('_'); + + let obj = (config = config ?? {}); + for (const [index, part] of keyParts.entries()) { + if (!CONFIG_KEY_PART_PATTERN.test(part)) { + throw new TypeError(`Invalid env config key '${key}'`); + } + if (index < keyParts.length - 1) { + obj = (obj[part] = obj[part] ?? {}) as JsonObject; + if (typeof obj !== 'object' || Array.isArray(obj)) { + const subKey = keyParts.slice(0, index + 1).join('_'); + throw new TypeError( + `Could not nest config for key '${key}' under existing value '${subKey}'`, + ); + } + } else { + if (part in obj) { + throw new TypeError( + `Refusing to override existing config at key '${key}'`, + ); + } + try { + const parsedValue = JSON.parse(value); + if (parsedValue === null) { + throw new Error('value may not be null'); + } + obj[part] = parsedValue; + } catch (error) { + throw new TypeError( + `Failed to parse JSON-serialized config value for key '${key}', ${error}`, + ); + } + } + } + } + } + + return config ? [config] : []; +} diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts new file mode 100644 index 0000000000..9c4a2d9f91 --- /dev/null +++ b/packages/config-loader/src/lib/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { findRootPath } from './paths'; +export { readConfigFile } from './reader'; +export { readEnv } from './env'; diff --git a/packages/config-loader/src/paths.test.ts b/packages/config-loader/src/lib/paths.test.ts similarity index 100% rename from packages/config-loader/src/paths.test.ts rename to packages/config-loader/src/lib/paths.test.ts diff --git a/packages/config-loader/src/paths.ts b/packages/config-loader/src/lib/paths.ts similarity index 100% rename from packages/config-loader/src/paths.ts rename to packages/config-loader/src/lib/paths.ts diff --git a/packages/config-loader/src/lib/reader.ts b/packages/config-loader/src/lib/reader.ts new file mode 100644 index 0000000000..160e45417e --- /dev/null +++ b/packages/config-loader/src/lib/reader.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yaml from 'yaml'; +import { isObject } from './utils'; +import { readSecret } from './secrets'; +import { JsonValue, JsonObject } from '@backstage/config'; +import { ReaderContext } from './types'; + +export async function readConfigFile(filePath: string, ctx: ReaderContext) { + const configYaml = await ctx.readFile(filePath); + const config = yaml.parse(configYaml); + + async function transform( + obj: JsonValue, + path: string, + ): Promise { + if (typeof obj !== 'object') { + return obj; + } else if (obj === null) { + return obj; + } else if (Array.isArray(obj)) { + const arr = new Array(); + + for (const [index, value] of obj.entries()) { + const out = await transform(value, `${path}[${index}]`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr; + } + + if ('$secret' in obj) { + if (!isObject(obj.$secret)) { + throw TypeError(`Expected object at secret ${path}.$secret`); + } + + try { + return await readSecret(obj.$secret, ctx); + } catch (error) { + throw new Error(`Invalid secret at ${path}: ${error.message}`); + } + } + + const out: JsonObject = {}; + + for (const [key, value] of Object.entries(obj)) { + const result = await transform(value, `${path}.${key}`); + if (result !== undefined) { + out[key] = result; + } + } + + return out; + } + + const finalConfig = await transform(config, ''); + if (!isObject(finalConfig)) { + throw new TypeError('Expected object at config root'); + } + return finalConfig; +} diff --git a/packages/config-loader/src/lib/secrets.ts b/packages/config-loader/src/lib/secrets.ts new file mode 100644 index 0000000000..797b9d1f44 --- /dev/null +++ b/packages/config-loader/src/lib/secrets.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import yaml from 'yaml'; +import { extname } from 'path'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { isObject, isNever } from './utils'; +import { ReaderContext } from './types'; + +type FileSecret = { + file: string; +}; + +type EnvSecret = { + env: string; +}; + +type DataSecret = { + data: string; + path: string | string[]; +}; + +type Secret = FileSecret | EnvSecret | DataSecret; + +const secretLoaderSchemas = { + file: yup.object({ + file: yup.string().required(), + }), + env: yup.object({ + env: yup.string().required(), + }), + data: yup.object({ + data: yup.string().required(), + path: yup.lazy(value => { + if (typeof value === 'string') { + return yup.string().required(); + } + return yup.array().of(yup.string().required()).required(); + }), + }), +}; + +const secretSchema = yup.lazy(value => { + if (typeof value !== 'object' || value === null) { + return yup.object().required(); + } + + const loaderTypes = Object.keys( + secretLoaderSchemas, + ) as (keyof typeof secretLoaderSchemas)[]; + + for (const key of loaderTypes) { + if (key in value) { + return secretLoaderSchemas[key]; + } + } + throw new yup.ValidationError( + `Secret must contain one of '${loaderTypes.join("', '")}'`, + value, + '$secret', + ); +}); + +const dataSecretParser: { + [ext in string]: (content: string) => Promise; +} = { + '.json': async content => JSON.parse(content), + '.yaml': async content => yaml.parse(content), + '.yml': async content => yaml.parse(content), +}; + +export async function readSecret( + data: JsonObject, + ctx: ReaderContext, +): Promise { + if (!ctx.shouldReadSecrets) { + return undefined; + } + + const secret = secretSchema.validateSync(data) as Secret; + + if ('file' in secret) { + return ctx.readFile(secret.file); + } + if ('env' in secret) { + return ctx.env[secret.env]; + } + if ('data' in secret) { + const ext = extname(secret.data); + const parser = dataSecretParser[ext]; + if (!parser) { + throw new Error(`No data secret parser available for extension ${ext}`); + } + + const content = await ctx.readFile(secret.data); + + const { path } = secret; + const parts = typeof path === 'string' ? path.split('.') : path; + + let value: JsonValue = await parser(content); + for (const [index, part] of parts.entries()) { + if (!isObject(value)) { + const errPath = parts.slice(0, index).join('.'); + throw new Error( + `Value is not an object at ${errPath} in ${secret.data}`, + ); + } + value = value[part]; + } + return String(value); + } + + isNever(); + throw new Error('Secret was left unhandled'); +} diff --git a/packages/config-loader/src/types.ts b/packages/config-loader/src/lib/types.ts similarity index 73% rename from packages/config-loader/src/types.ts rename to packages/config-loader/src/lib/types.ts index 0c27ab1854..bde324a3c1 100644 --- a/packages/config-loader/src/types.ts +++ b/packages/config-loader/src/lib/types.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -export type LoadConfigOptions = { - // Config path, defaults to app-config.yaml in project root - configPath?: string; +export type ReadFileFunc = (path: string) => Promise; - // Whether to read secrets or omit them, defaults to false. - shouldReadSecrets?: boolean; +export type ReaderContext = { + shouldReadSecrets: boolean; + readFile: ReadFileFunc; + env: { [name in string]?: string }; }; diff --git a/packages/config-loader/src/lib/utils.ts b/packages/config-loader/src/lib/utils.ts new file mode 100644 index 0000000000..37145ec971 --- /dev/null +++ b/packages/config-loader/src/lib/utils.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonValue, JsonObject } from '@backstage/config'; + +export function isObject(obj: JsonValue | undefined): obj is JsonObject { + if (typeof obj !== 'object') { + return false; + } else if (Array.isArray(obj)) { + return false; + } + return obj !== null; +} + +// A thing to make sure we've narrowed the type down to never +export function isNever() { + return void 0 as T; +} diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index ba577c4637..a8adbde32c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -15,254 +15,18 @@ */ import fs from 'fs-extra'; -import * as yup from 'yup'; -import yaml from 'yaml'; -import { resolve as resolvePath, dirname, extname } from 'path'; -import { AppConfig, JsonObject, JsonValue } from '@backstage/config'; -import { findRootPath } from './paths'; -import { LoadConfigOptions } from './types'; +import { resolve as resolvePath, dirname } from 'path'; +import { AppConfig } from '@backstage/config'; +import { findRootPath, readConfigFile, readEnv } from './lib'; -const ENV_PREFIX = 'APP_CONFIG_'; +export type LoadConfigOptions = { + // Config path, defaults to app-config.yaml in project root + configPath?: string; -// Update the same pattern in config package if this is changed -const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i; - -export function readEnv(env: { - [name: string]: string | undefined; -}): AppConfig[] { - let config: JsonObject | undefined = undefined; - - for (const [name, value] of Object.entries(env)) { - if (!value) { - continue; - } - if (name.startsWith(ENV_PREFIX)) { - const key = name.replace(ENV_PREFIX, ''); - const keyParts = key.split('_'); - - let obj = (config = config ?? {}); - for (const [index, part] of keyParts.entries()) { - if (!CONFIG_KEY_PART_PATTERN.test(part)) { - throw new TypeError(`Invalid env config key '${key}'`); - } - if (index < keyParts.length - 1) { - obj = (obj[part] = obj[part] ?? {}) as JsonObject; - if (typeof obj !== 'object' || Array.isArray(obj)) { - const subKey = keyParts.slice(0, index + 1).join('_'); - throw new TypeError( - `Could not nest config for key '${key}' under existing value '${subKey}'`, - ); - } - } else { - if (part in obj) { - throw new TypeError( - `Refusing to override existing config at key '${key}'`, - ); - } - try { - const parsedValue = JSON.parse(value); - if (parsedValue === null) { - throw new Error('value may not be null'); - } - obj[part] = parsedValue; - } catch (error) { - throw new TypeError( - `Failed to parse JSON-serialized config value for key '${key}', ${error}`, - ); - } - } - } - } - } - - return config ? [config] : []; -} - -type ReadFileFunc = (path: string) => Promise; - -type ReaderContext = { - shouldReadSecrets: boolean; - readFile: ReadFileFunc; - env: { [name in string]?: string }; + // Whether to read secrets or omit them, defaults to false. + shouldReadSecrets?: boolean; }; -function isObject(obj: JsonValue | undefined): obj is JsonObject { - if (typeof obj !== 'object') { - return false; - } else if (Array.isArray(obj)) { - return false; - } - return obj !== null; -} - -type FileSecret = { - file: string; -}; - -type EnvSecret = { - env: string; -}; - -type DataSecret = { - data: string; - path: string | string[]; -}; - -type Secret = FileSecret | EnvSecret | DataSecret; - -const secretLoaderSchemas = { - file: yup.object({ - file: yup.string().required(), - }), - env: yup.object({ - env: yup.string().required(), - }), - data: yup.object({ - data: yup.string().required(), - path: yup.lazy(value => { - if (typeof value === 'string') { - return yup.string().required(); - } - return yup.array().of(yup.string().required()).required(); - }), - }), -}; - -const secretSchema = yup.lazy(value => { - if (typeof value !== 'object' || value === null) { - return yup.object().required(); - } - - const loaderTypes = Object.keys( - secretLoaderSchemas, - ) as (keyof typeof secretLoaderSchemas)[]; - - for (const key of loaderTypes) { - if (key in value) { - return secretLoaderSchemas[key]; - } - } - throw new yup.ValidationError( - `Secret must contain one of '${loaderTypes.join("', '")}'`, - value, - '$secret', - ); -}); - -const dataSecretParser: { - [ext in string]: (content: string) => Promise; -} = { - '.json': async content => JSON.parse(content), - '.yaml': async content => yaml.parse(content), - '.yml': async content => yaml.parse(content), -}; - -// A thing to make sure we've narrowed the type down to never -function isNever() { - return void 0 as T; -} - -export async function readSecret( - data: JsonObject, - ctx: ReaderContext, -): Promise { - if (!ctx.shouldReadSecrets) { - return undefined; - } - - const secret = secretSchema.validateSync(data) as Secret; - - if ('file' in secret) { - return ctx.readFile(secret.file); - } - if ('env' in secret) { - return ctx.env[secret.env]; - } - if ('data' in secret) { - const ext = extname(secret.data); - const parser = dataSecretParser[ext]; - if (!parser) { - throw new Error(`No data secret parser available for extension ${ext}`); - } - - const content = await ctx.readFile(secret.data); - - const { path } = secret; - const parts = typeof path === 'string' ? path.split('.') : path; - - let value: JsonValue = await parser(content); - for (const [index, part] of parts.entries()) { - if (!isObject(value)) { - const errPath = parts.slice(0, index).join('.'); - throw new Error( - `Value is not an object at ${errPath} in ${secret.data}`, - ); - } - value = value[part]; - } - return String(value); - } - - isNever(); - throw new Error('Secret was left unhandled'); -} - -export async function readConfigFile(filePath: string, ctx: ReaderContext) { - const configYaml = await ctx.readFile(filePath); - const config = yaml.parse(configYaml); - - async function transform( - obj: JsonValue, - path: string, - ): Promise { - if (typeof obj !== 'object') { - return obj; - } else if (obj === null) { - return obj; - } else if (Array.isArray(obj)) { - const arr = new Array(); - - for (const [index, value] of obj.entries()) { - const out = await transform(value, `${path}[${index}]`); - if (out !== undefined) { - arr.push(out); - } - } - - return arr; - } - - if ('$secret' in obj) { - if (!isObject(obj.$secret)) { - throw TypeError(`Expected object at secret ${path}.$secret`); - } - - try { - return await readSecret(obj.$secret, ctx); - } catch (error) { - throw new Error(`Invalid secret at ${path}: ${error.message}`); - } - } - - const out: JsonObject = {}; - - for (const [key, value] of Object.entries(obj)) { - const result = await transform(value, `${path}.${key}`); - if (result !== undefined) { - out[key] = result; - } - } - - return out; - } - - const finalConfig = await transform(config, ''); - if (!isObject(finalConfig)) { - throw new TypeError('Expected object at config root'); - } - return finalConfig; -} - export async function loadStaticConfig( options: LoadConfigOptions, ): Promise {