config-loader: refactor to use transforms as a base building block
This commit is contained in:
Patrik Oldsberg
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { createIncludeTransform } from './include';
|
||||
|
||||
const env = jest.fn(async (name: string) => {
|
||||
return ({
|
||||
SECRET: 'my-secret',
|
||||
} as { [name: string]: string })[name];
|
||||
});
|
||||
|
||||
const readFile = jest.fn(async (path: string) => {
|
||||
const content = ({
|
||||
'my-secret': 'secret',
|
||||
'my-data.json': '{"a":{"b":{"c":42}}}',
|
||||
'my-data.yaml': 'some:\n yaml:\n key: 7',
|
||||
'my-data.yml': 'different: { key: hello }',
|
||||
'invalid.yaml': 'foo: [}',
|
||||
} as { [key: string]: string })[path];
|
||||
|
||||
if (!content) {
|
||||
throw new Error('File not found!');
|
||||
}
|
||||
return content;
|
||||
});
|
||||
|
||||
const includeTransform = createIncludeTransform(env, readFile);
|
||||
|
||||
describe('includeTransform', () => {
|
||||
it('should not transform unknown values', async () => {
|
||||
await expect(includeTransform('foo')).resolves.toEqual([
|
||||
false,
|
||||
expect.anything(),
|
||||
]);
|
||||
await expect(includeTransform([1])).resolves.toEqual([
|
||||
false,
|
||||
expect.anything(),
|
||||
]);
|
||||
await expect(includeTransform(1)).resolves.toEqual([
|
||||
false,
|
||||
expect.anything(),
|
||||
]);
|
||||
await expect(includeTransform({ x: 'y' })).resolves.toEqual([
|
||||
false,
|
||||
expect.anything(),
|
||||
]);
|
||||
await expect(includeTransform(null)).resolves.toEqual([false, null]);
|
||||
});
|
||||
|
||||
it('should include text files', async () => {
|
||||
await expect(includeTransform({ $file: 'my-secret' })).resolves.toEqual([
|
||||
true,
|
||||
'secret',
|
||||
]);
|
||||
await expect(includeTransform({ $file: 'no-secret' })).rejects.toThrow(
|
||||
'File not found!',
|
||||
);
|
||||
});
|
||||
|
||||
it('should include env vars', async () => {
|
||||
await expect(includeTransform({ $env: 'SECRET' })).resolves.toEqual([
|
||||
true,
|
||||
'my-secret',
|
||||
]);
|
||||
await expect(includeTransform({ $env: 'NO_SECRET' })).resolves.toEqual([
|
||||
true,
|
||||
undefined,
|
||||
]);
|
||||
});
|
||||
|
||||
it('should include config files', async () => {
|
||||
// New format with path in fragment
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.json#a.b.c' }),
|
||||
).resolves.toEqual([true, 42]);
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.json#a.b' }),
|
||||
).resolves.toEqual([true, { c: 42 }]);
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.yaml#some.yaml.key' }),
|
||||
).resolves.toEqual([true, 7]);
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.yaml' }),
|
||||
).resolves.toEqual([
|
||||
true,
|
||||
{
|
||||
some: { yaml: { key: 7 } },
|
||||
},
|
||||
]);
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.yaml#' }),
|
||||
).resolves.toEqual([
|
||||
true,
|
||||
{
|
||||
some: { yaml: { key: 7 } },
|
||||
},
|
||||
]);
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.yml#different.key' }),
|
||||
).resolves.toEqual([true, 'hello']);
|
||||
});
|
||||
|
||||
it('should reject invalid includes', async () => {
|
||||
await expect(
|
||||
includeTransform({ $include: 'no-parser.js' }),
|
||||
).rejects.toThrow('no configuration parser available for extension .js');
|
||||
await expect(
|
||||
includeTransform({ $include: 'no-data.yml#different.key' }),
|
||||
).rejects.toThrow('File not found!');
|
||||
await expect(
|
||||
includeTransform({ $include: 'my-data.yml#missing.key' }),
|
||||
).rejects.toThrow(
|
||||
"value at 'missing' in included file my-data.yml is not an object",
|
||||
);
|
||||
await expect(
|
||||
includeTransform({ $include: 'invalid.yaml' }),
|
||||
).rejects.toThrow(
|
||||
'failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import yaml from 'yaml';
|
||||
import { extname } from 'path';
|
||||
import { JsonObject, JsonValue } from '@backstage/config';
|
||||
import { isObject } from './utils';
|
||||
import { TransformFunc, EnvFunc, ReadFileFunc } from './types';
|
||||
|
||||
// Parsers for each type of included file
|
||||
const includeFileParser: {
|
||||
[ext in string]: (content: string) => Promise<JsonObject>;
|
||||
} = {
|
||||
'.json': async content => JSON.parse(content),
|
||||
'.yaml': async content => yaml.parse(content),
|
||||
'.yml': async content => yaml.parse(content),
|
||||
};
|
||||
|
||||
/**
|
||||
* Transforms a secret description into the actual secret value.
|
||||
*/
|
||||
export function createIncludeTransform(
|
||||
env: EnvFunc,
|
||||
readFile: ReadFileFunc,
|
||||
): TransformFunc {
|
||||
return async (input: JsonValue) => {
|
||||
if (!isObject(input)) {
|
||||
return [false, input];
|
||||
}
|
||||
// Check if there's any key that starts with a '$', in that case we treat
|
||||
// this entire object as a secret.
|
||||
const [secretKey] = Object.keys(input).filter(key => key.startsWith('$'));
|
||||
if (secretKey) {
|
||||
if (Object.keys(input).length !== 1) {
|
||||
throw new Error(
|
||||
`include key ${secretKey} should not have adjacent keys`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return [false, input];
|
||||
}
|
||||
|
||||
const secretValue = input[secretKey];
|
||||
if (typeof secretValue !== 'string') {
|
||||
throw new Error(`${secretKey} include value is not a string`);
|
||||
}
|
||||
|
||||
switch (secretKey) {
|
||||
case '$file':
|
||||
try {
|
||||
return [true, await readFile(secretValue)];
|
||||
} catch (error) {
|
||||
throw new Error(`failed to read file ${secretValue}, ${error}`);
|
||||
}
|
||||
case '$env':
|
||||
try {
|
||||
return [true, await env(secretValue)];
|
||||
} catch (error) {
|
||||
throw new Error(`failed to read env ${secretValue}, ${error}`);
|
||||
}
|
||||
|
||||
case '$include': {
|
||||
const [filePath, dataPath] = secretValue.split(/#(.*)/);
|
||||
|
||||
const ext = extname(filePath);
|
||||
const parser = includeFileParser[ext];
|
||||
if (!parser) {
|
||||
throw new Error(
|
||||
`no configuration parser available for included file ${filePath}`,
|
||||
);
|
||||
}
|
||||
|
||||
const content = await readFile(filePath);
|
||||
|
||||
const parts = dataPath ? dataPath.split('.') : [];
|
||||
|
||||
let value: JsonValue | undefined;
|
||||
try {
|
||||
value = await parser(content);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`failed to parse included file ${filePath}, ${error}`,
|
||||
);
|
||||
}
|
||||
|
||||
// This bit handles selecting a subtree in the included file, if a path was provided after a #
|
||||
for (const [index, part] of parts.entries()) {
|
||||
if (!isObject(value)) {
|
||||
const errPath = parts.slice(0, index).join('.');
|
||||
throw new Error(
|
||||
`value at '${errPath}' in included file ${filePath} is not an object`,
|
||||
);
|
||||
}
|
||||
value = value[part];
|
||||
}
|
||||
|
||||
return [true, value];
|
||||
}
|
||||
|
||||
default:
|
||||
throw new Error(`unknown secret ${secretKey}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { readConfigFile } from './reader';
|
||||
export { applyConfigTransforms } from './transform';
|
||||
export { readEnvConfig } from './env';
|
||||
export { readSecret } from './secrets';
|
||||
export { createIncludeTransform } from './include';
|
||||
export * from './schema';
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { readConfigFile } from './reader';
|
||||
import { ReaderContext, ReadSecretFunc } from './types';
|
||||
|
||||
function memoryFiles(files: { [path: string]: string }) {
|
||||
return async (path: string) => {
|
||||
if (path in files) {
|
||||
return files[path];
|
||||
}
|
||||
throw new Error(`File not found, ${path}`);
|
||||
};
|
||||
}
|
||||
|
||||
const mockContext: ReaderContext = {
|
||||
env: {},
|
||||
readFile: jest.fn(),
|
||||
readSecret: jest.fn(),
|
||||
};
|
||||
|
||||
describe('readConfigFile', () => {
|
||||
it('should read a plain config file', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml':
|
||||
'app: { title: "Test", x: 1, y: [null, true], z: null }',
|
||||
});
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
});
|
||||
|
||||
await expect(config).resolves.toEqual({
|
||||
data: {
|
||||
app: {
|
||||
title: 'Test',
|
||||
x: 1,
|
||||
y: [true],
|
||||
},
|
||||
},
|
||||
context: 'app-config.yaml',
|
||||
});
|
||||
});
|
||||
|
||||
it('should error out if the config file has invalid syntax', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml': 'app: { title: ]',
|
||||
});
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
});
|
||||
|
||||
await expect(config).rejects.toThrow('Flow map contains an unexpected ]');
|
||||
});
|
||||
|
||||
it('should error out if config is not an object', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml': '[]',
|
||||
});
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
});
|
||||
|
||||
await expect(config).rejects.toThrow('Expected object at config root');
|
||||
});
|
||||
|
||||
it('should read secrets', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml': 'app: { $file: "./my-secret" }',
|
||||
});
|
||||
const readSecret = jest.fn().mockResolvedValue('secret');
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
readSecret: readSecret as ReadSecretFunc,
|
||||
});
|
||||
|
||||
await expect(config).resolves.toEqual({
|
||||
data: {
|
||||
app: 'secret',
|
||||
},
|
||||
context: 'app-config.yaml',
|
||||
});
|
||||
expect(readSecret).toHaveBeenCalledWith('.app', {
|
||||
file: './my-secret',
|
||||
});
|
||||
});
|
||||
|
||||
it('should not allow keys adjacent to secrets', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml': 'app: { extraKey: 3, $file: "./my-secret" }',
|
||||
});
|
||||
const readSecret = jest.fn().mockResolvedValue('secret');
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
readSecret: readSecret as ReadSecretFunc,
|
||||
});
|
||||
|
||||
await expect(config).rejects.toThrow(
|
||||
"Secret key '$file' has adjacent keys at .app",
|
||||
);
|
||||
expect(readSecret).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should forward secret reading errors', async () => {
|
||||
const readFile = memoryFiles({
|
||||
'./app-config.yaml': 'app: { $file: {} }',
|
||||
});
|
||||
const readSecret = jest.fn().mockRejectedValue(new Error('NOPE'));
|
||||
|
||||
const config = readConfigFile('./app-config.yaml', {
|
||||
...mockContext,
|
||||
readFile,
|
||||
readSecret: readSecret as ReadSecretFunc,
|
||||
});
|
||||
|
||||
await expect(config).rejects.toThrow('Invalid secret at .app: NOPE');
|
||||
});
|
||||
});
|
||||
@@ -1,129 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { readSecret } from './secrets';
|
||||
import { ReaderContext } from './types';
|
||||
|
||||
const ctx: ReaderContext = {
|
||||
env: {
|
||||
SECRET: 'my-secret',
|
||||
},
|
||||
readSecret: jest.fn(),
|
||||
async readFile(path) {
|
||||
const content = ({
|
||||
'my-secret': 'secret',
|
||||
'my-data.json': '{"a":{"b":{"c":42}}}',
|
||||
'my-data.yaml': 'some:\n yaml:\n key: 7',
|
||||
'my-data.yml': 'different: { key: hello }',
|
||||
'invalid.yaml': 'foo: [}',
|
||||
} as { [key: string]: string })[path];
|
||||
|
||||
if (!content) {
|
||||
throw new Error('File not found!');
|
||||
}
|
||||
return content;
|
||||
},
|
||||
};
|
||||
|
||||
describe('readSecret', () => {
|
||||
it('should read file secrets', async () => {
|
||||
await expect(readSecret({ file: 'my-secret' }, ctx)).resolves.toBe(
|
||||
'secret',
|
||||
);
|
||||
await expect(readSecret({ file: 'no-secret' }, ctx)).rejects.toThrow(
|
||||
'File not found!',
|
||||
);
|
||||
});
|
||||
|
||||
it('should read present env secrets', async () => {
|
||||
await expect(readSecret({ env: 'SECRET' }, ctx)).resolves.toBe('my-secret');
|
||||
await expect(readSecret({ env: 'NO_SECRET' }, ctx)).resolves.toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('should include extra files', async () => {
|
||||
// New format with path in fragment
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.json#a.b.c' }, ctx),
|
||||
).resolves.toBe(42);
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.json#a.b' }, ctx),
|
||||
).resolves.toEqual({ c: 42 });
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.yaml#some.yaml.key' }, ctx),
|
||||
).resolves.toBe(7);
|
||||
await expect(readSecret({ include: 'my-data.yaml' }, ctx)).resolves.toEqual(
|
||||
{
|
||||
some: { yaml: { key: 7 } },
|
||||
},
|
||||
);
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.yaml#' }, ctx),
|
||||
).resolves.toEqual({
|
||||
some: { yaml: { key: 7 } },
|
||||
});
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.yml#different.key' }, ctx),
|
||||
).resolves.toBe('hello');
|
||||
await expect(
|
||||
readSecret({ include: 'no-data.yml#different.key' }, ctx),
|
||||
).rejects.toThrow('File not found!');
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.yml#missing.key' }, ctx),
|
||||
).rejects.toThrow('Value is not an object at missing in my-data.yml');
|
||||
await expect(readSecret({ include: 'invalid.yaml' }, ctx)).rejects.toThrow(
|
||||
'Failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }',
|
||||
);
|
||||
});
|
||||
|
||||
it('should reject invalid secrets', async () => {
|
||||
await expect(readSecret('hello' as any, ctx)).rejects.toThrow(
|
||||
'secret must be a `object` type, but the final value was: `"hello"`.',
|
||||
);
|
||||
await expect(readSecret({}, ctx)).rejects.toThrow(
|
||||
"Secret must contain one of 'file', 'env', 'include'",
|
||||
);
|
||||
await expect(readSecret({ unknown: 'derp' }, ctx)).rejects.toThrow(
|
||||
"Secret must contain one of 'file', 'env', 'include'",
|
||||
);
|
||||
await expect(readSecret({ include: 'no-parser.js' }, ctx)).rejects.toThrow(
|
||||
'No data secret parser available for extension .js',
|
||||
);
|
||||
await expect(
|
||||
readSecret({ include: 'my-data.yaml#some.wrong.yaml.key' }, ctx),
|
||||
).rejects.toThrow('Value is not an object at some.wrong in my-data.yaml');
|
||||
});
|
||||
|
||||
it('should have 100% test coverage', async () => {
|
||||
let firstVisit = true;
|
||||
const secret = {};
|
||||
const proto = {
|
||||
get file() {
|
||||
if (!firstVisit) {
|
||||
Object.setPrototypeOf(secret, {});
|
||||
}
|
||||
firstVisit = false;
|
||||
return 'a-file';
|
||||
},
|
||||
};
|
||||
Object.setPrototypeOf(secret, proto);
|
||||
|
||||
await expect(readSecret(secret, ctx)).rejects.toThrow(
|
||||
'Secret was left unhandled',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,134 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import * as yup from 'yup';
|
||||
import yaml from 'yaml';
|
||||
import { extname } from 'path';
|
||||
import { JsonObject, JsonValue } from '@backstage/config';
|
||||
import { isObject, isNever } from './utils';
|
||||
import { ReaderContext } from './types';
|
||||
|
||||
// Reads a file and forwards the contents as is, assuming ut8 encoding
|
||||
type FileSecret = {
|
||||
// Path to the secret file, relative to the config file.
|
||||
file: string;
|
||||
};
|
||||
|
||||
// Reads the secret from an environment variable.
|
||||
type EnvSecret = {
|
||||
// The name of the environment file.
|
||||
env: string;
|
||||
};
|
||||
|
||||
// TODO(Rugvip): Move this out of secret reading when we remove the deprecated DataSecret and $secret format
|
||||
type IncludeSecret = {
|
||||
include: string;
|
||||
};
|
||||
|
||||
type Secret = FileSecret | EnvSecret | IncludeSecret;
|
||||
|
||||
// Schema for each type of secret description
|
||||
const secretLoaderSchemas = {
|
||||
file: yup.object({
|
||||
file: yup.string().required(),
|
||||
}),
|
||||
env: yup.object({
|
||||
env: yup.string().required(),
|
||||
}),
|
||||
include: yup.object({
|
||||
include: yup.string().required(),
|
||||
}),
|
||||
};
|
||||
|
||||
// The top-level secret schema, which figures out what type of secret it is.
|
||||
const secretSchema = yup.lazy<object | undefined>(value => {
|
||||
if (typeof value !== 'object' || value === null) {
|
||||
return yup.object().required().label('secret');
|
||||
}
|
||||
|
||||
const loaderTypes = Object.keys(
|
||||
secretLoaderSchemas,
|
||||
) as (keyof typeof secretLoaderSchemas)[];
|
||||
|
||||
for (const key of loaderTypes) {
|
||||
if (key in value) {
|
||||
return secretLoaderSchemas[key];
|
||||
}
|
||||
}
|
||||
throw new yup.ValidationError(
|
||||
`Secret must contain one of '${loaderTypes.join("', '")}'`,
|
||||
value,
|
||||
'$secret',
|
||||
);
|
||||
});
|
||||
|
||||
// Parsers for each type of data secret file.
|
||||
const dataSecretParser: {
|
||||
[ext in string]: (content: string) => Promise<JsonObject>;
|
||||
} = {
|
||||
'.json': async content => JSON.parse(content),
|
||||
'.yaml': async content => yaml.parse(content),
|
||||
'.yml': async content => yaml.parse(content),
|
||||
};
|
||||
|
||||
/**
|
||||
* Transforms a secret description into the actual secret value.
|
||||
*/
|
||||
export async function readSecret(
|
||||
data: JsonObject,
|
||||
ctx: ReaderContext,
|
||||
): Promise<JsonValue | undefined> {
|
||||
const secret = secretSchema.validateSync(data, { strict: true }) as Secret;
|
||||
|
||||
if ('file' in secret) {
|
||||
return ctx.readFile(secret.file);
|
||||
}
|
||||
if ('env' in secret) {
|
||||
return ctx.env[secret.env];
|
||||
}
|
||||
if ('include' in secret) {
|
||||
const [filePath, dataPath] = secret.include.split(/#(.*)/);
|
||||
|
||||
const ext = extname(filePath);
|
||||
const parser = dataSecretParser[ext];
|
||||
if (!parser) {
|
||||
throw new Error(`No data secret parser available for extension ${ext}`);
|
||||
}
|
||||
|
||||
const content = await ctx.readFile(filePath);
|
||||
|
||||
const parts = dataPath ? dataPath.split('.') : [];
|
||||
|
||||
let value: JsonValue | undefined;
|
||||
try {
|
||||
value = await parser(content);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to parse included file ${filePath}, ${error}`);
|
||||
}
|
||||
for (const [index, part] of parts.entries()) {
|
||||
if (!isObject(value)) {
|
||||
const errPath = parts.slice(0, index).join('.');
|
||||
throw new Error(`Value is not an object at ${errPath} in ${filePath}`);
|
||||
}
|
||||
value = value[part];
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
isNever<typeof secret>();
|
||||
throw new Error('Secret was left unhandled');
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { applyConfigTransforms } from './transform';
|
||||
|
||||
describe('applyConfigTransforms', () => {
|
||||
it('should apply not transforms to input', async () => {
|
||||
const data = applyConfigTransforms(
|
||||
{
|
||||
app: {
|
||||
title: 'Test',
|
||||
x: 1,
|
||||
y: [null, true],
|
||||
z: null,
|
||||
},
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
await expect(data).resolves.toEqual({
|
||||
app: {
|
||||
title: 'Test',
|
||||
x: 1,
|
||||
y: [true],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw if input is not an object', async () => {
|
||||
const config = applyConfigTransforms('not-config', []);
|
||||
|
||||
await expect(config).rejects.toThrow('expected object at config root');
|
||||
});
|
||||
|
||||
it('should apply transforms', async () => {
|
||||
const config = applyConfigTransforms(
|
||||
{
|
||||
app: {
|
||||
title: 'Test',
|
||||
x: 1,
|
||||
y: [null, true],
|
||||
z: null,
|
||||
},
|
||||
},
|
||||
[
|
||||
async value => {
|
||||
if (typeof value === 'number') {
|
||||
return [true, value + 1];
|
||||
}
|
||||
return [false, value];
|
||||
},
|
||||
async value => {
|
||||
if (typeof value === 'string' && value.length > 1) {
|
||||
return [true, value.split('')];
|
||||
}
|
||||
return [false, value];
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
await expect(config).resolves.toEqual({
|
||||
app: {
|
||||
title: ['T', 'e', 's', 't'],
|
||||
x: 2,
|
||||
y: [true],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
+28
-36
@@ -14,29 +14,39 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
|
||||
import { basename } from 'path';
|
||||
import yaml from 'yaml';
|
||||
import { ReaderContext } from './types';
|
||||
import { JsonObject, JsonValue } from '@backstage/config';
|
||||
import { TransformFunc } from './types';
|
||||
import { isObject } from './utils';
|
||||
|
||||
/**
|
||||
* Reads and parses, and validates, and transforms a single config file.
|
||||
* The transformation rewrites any special values, like the $secret key.
|
||||
*/
|
||||
export async function readConfigFile(
|
||||
filePath: string,
|
||||
ctx: ReaderContext,
|
||||
): Promise<AppConfig> {
|
||||
const configYaml = await ctx.readFile(filePath);
|
||||
const config = yaml.parse(configYaml);
|
||||
|
||||
const context = basename(filePath);
|
||||
|
||||
export async function applyConfigTransforms(
|
||||
input: JsonValue,
|
||||
transforms: TransformFunc[],
|
||||
): Promise<JsonObject> {
|
||||
async function transform(
|
||||
obj: JsonValue,
|
||||
inputObj: JsonValue,
|
||||
path: string,
|
||||
): Promise<JsonValue | undefined> {
|
||||
let obj = inputObj;
|
||||
|
||||
for (const tf of transforms) {
|
||||
try {
|
||||
const [applied, newObj] = await tf(inputObj);
|
||||
if (applied) {
|
||||
if (newObj === undefined) {
|
||||
return newObj;
|
||||
}
|
||||
obj = newObj;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`error at ${path}, ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof obj !== 'object') {
|
||||
return obj;
|
||||
} else if (obj === null) {
|
||||
@@ -54,24 +64,6 @@ export async function readConfigFile(
|
||||
return arr;
|
||||
}
|
||||
|
||||
// Check if there's any key that starts with a '$', in that case we treat
|
||||
// this entire object as a secret.
|
||||
const [secretKey] = Object.keys(obj).filter(key => key.startsWith('$'));
|
||||
if (secretKey) {
|
||||
if (Object.keys(obj).length !== 1) {
|
||||
throw new Error(
|
||||
`Secret key '${secretKey}' has adjacent keys at ${path}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await ctx.readSecret(path, {
|
||||
[secretKey.slice(1)]: obj[secretKey],
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`Invalid secret at ${path}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const out: JsonObject = {};
|
||||
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
@@ -87,9 +79,9 @@ export async function readConfigFile(
|
||||
return out;
|
||||
}
|
||||
|
||||
const finalConfig = await transform(config, '');
|
||||
if (!isObject(finalConfig)) {
|
||||
throw new TypeError('Expected object at config root');
|
||||
const finalData = await transform(input, '');
|
||||
if (!isObject(finalData)) {
|
||||
throw new TypeError('expected object at config root');
|
||||
}
|
||||
return { data: finalConfig, context };
|
||||
return finalData;
|
||||
}
|
||||
@@ -14,20 +14,12 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { JsonObject, JsonValue } from '@backstage/config';
|
||||
import { JsonValue } from '@backstage/config';
|
||||
|
||||
export type EnvFunc = (name: string) => Promise<string | undefined>;
|
||||
|
||||
export type ReadFileFunc = (path: string) => Promise<string>;
|
||||
export type ReadSecretFunc = (
|
||||
path: string,
|
||||
desc: JsonObject,
|
||||
) => Promise<JsonValue | undefined>;
|
||||
export type SkipFunc = (path: string) => boolean;
|
||||
|
||||
/**
|
||||
* Common context that provides all the necessary hooks for reading configuration files.
|
||||
*/
|
||||
export type ReaderContext = {
|
||||
env: { [name in string]?: string };
|
||||
readFile: ReadFileFunc;
|
||||
readSecret: ReadSecretFunc;
|
||||
};
|
||||
export type TransformFunc = (
|
||||
value: JsonValue,
|
||||
) => Promise<[boolean, JsonValue | undefined]>;
|
||||
|
||||
@@ -24,8 +24,3 @@ export function isObject(obj: JsonValue | undefined): obj is JsonObject {
|
||||
}
|
||||
return obj !== null;
|
||||
}
|
||||
|
||||
// A thing to make sure we've narrowed the type down to never
|
||||
export function isNever<T extends never>() {
|
||||
return void 0 as T;
|
||||
}
|
||||
|
||||
@@ -15,9 +15,14 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import { resolve as resolvePath, dirname, isAbsolute } from 'path';
|
||||
import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
|
||||
import { readConfigFile, readEnvConfig, readSecret } from './lib';
|
||||
import yaml from 'yaml';
|
||||
import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path';
|
||||
import { AppConfig } from '@backstage/config';
|
||||
import {
|
||||
applyConfigTransforms,
|
||||
readEnvConfig,
|
||||
createIncludeTransform,
|
||||
} from './lib';
|
||||
|
||||
export type LoadConfigOptions = {
|
||||
// The root directory of the config loading context. Used to find default configs.
|
||||
@@ -30,30 +35,6 @@ export type LoadConfigOptions = {
|
||||
env: string;
|
||||
};
|
||||
|
||||
class Context {
|
||||
constructor(
|
||||
private readonly options: {
|
||||
env: { [name in string]?: string };
|
||||
rootPath: string;
|
||||
},
|
||||
) {}
|
||||
|
||||
get env() {
|
||||
return this.options.env;
|
||||
}
|
||||
|
||||
async readFile(path: string): Promise<string> {
|
||||
return fs.readFile(resolvePath(this.options.rootPath, path), 'utf8');
|
||||
}
|
||||
|
||||
async readSecret(
|
||||
_path: string,
|
||||
desc: JsonObject,
|
||||
): Promise<JsonValue | undefined> {
|
||||
return readSecret(desc, this);
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadConfig(
|
||||
options: LoadConfigOptions,
|
||||
): Promise<AppConfig[]> {
|
||||
@@ -82,24 +63,28 @@ export async function loadConfig(
|
||||
}
|
||||
}
|
||||
|
||||
const env = async (name: string) => process.env[name];
|
||||
|
||||
try {
|
||||
for (const configPath of configPaths) {
|
||||
if (!isAbsolute(configPath)) {
|
||||
throw new Error(`Config load path is not absolute: '${configPath}'`);
|
||||
}
|
||||
const config = await readConfigFile(
|
||||
configPath,
|
||||
new Context({
|
||||
env: process.env,
|
||||
rootPath: dirname(configPath),
|
||||
}),
|
||||
);
|
||||
|
||||
configs.push(config);
|
||||
const dir = dirname(configPath);
|
||||
const readFile = (path: string) =>
|
||||
fs.readFile(resolvePath(dir, path), 'utf8');
|
||||
|
||||
const input = yaml.parse(await readFile(configPath));
|
||||
const data = await applyConfigTransforms(input, [
|
||||
createIncludeTransform(env, readFile),
|
||||
]);
|
||||
|
||||
configs.push({ data, context: basename(configPath) });
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Failed to read static configuration file: ${error.message}`,
|
||||
`Failed to read static configuration file, ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user