config-loader: split up config schema implementation and move into separate lib

This commit is contained in:
Patrik Oldsberg
2020-11-08 14:18:58 +01:00
parent 4795278039
commit 8c65cec932
10 changed files with 384 additions and 295 deletions
+1 -1
View File
@@ -14,6 +14,6 @@
* limitations under the License.
*/
export { readEnvConfig } from './lib';
export { readEnvConfig, loadConfigSchema } from './lib';
export { loadConfig } from './loader';
export type { LoadConfigOptions } from './loader';
+1 -1
View File
@@ -17,4 +17,4 @@
export { readConfigFile } from './reader';
export { readEnvConfig } from './env';
export { readSecret } from './secrets';
export { loadSchema } from './schema';
export { loadConfigSchema } from './schema';
-289
View File
@@ -1,289 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { resolve as resolvePath, dirname } from 'path';
import Ajv from 'ajv';
import { JSONSchema7 as JSONSchema } from 'json-schema';
import mergeAllOf, { Resolvers } from 'json-schema-merge-allof';
import {
AppConfig,
ConfigReader,
JsonObject,
JsonValue,
} from '@backstage/config';
const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const;
type ConfigVisibility = typeof CONFIG_VISIBILITIES[number];
const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend';
type ConfigSchema = {
process(
appConfigs: AppConfig[],
options?: ConfigProcessingOptions,
): AppConfig[];
};
type ConfigSchemaPackageEntry = {
value: JSONSchema;
path: string;
};
type Options = {
dependencies: string[];
};
type ValidationError = string;
type ValidationResult = {
errors?: ValidationError[];
visibilityByPath: Map<string, ConfigVisibility>;
};
type ValidationFunc = (configs: AppConfig[]) => ValidationResult;
type ConfigProcessingOptions = {
visibilities?: ConfigVisibility[];
};
export function filterByVisibility(
data: JsonObject,
includeVisibilities: ConfigVisibility[],
visibilityByPath: Map<string, ConfigVisibility>,
): JsonObject {
function transform(jsonVal: JsonValue, path: string): JsonValue | undefined {
if (typeof jsonVal !== 'object') {
const visibility =
visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY;
if (includeVisibilities.includes(visibility)) {
return jsonVal;
}
return undefined;
} else if (jsonVal === null) {
return undefined;
} else if (Array.isArray(jsonVal)) {
const arr = new Array<JsonValue>();
for (const [index, value] of jsonVal.entries()) {
const out = transform(value, `${path}/${index}`);
if (out !== undefined) {
arr.push(out);
}
}
return arr.length === 0 ? undefined : arr;
}
const outObj: JsonObject = {};
for (const [key, value] of Object.entries(jsonVal)) {
if (value === undefined) {
continue;
}
const out = transform(value, `${path}/${key}`);
if (out !== undefined) {
outObj[key] = out;
}
}
return Object.keys(outObj).length === 0 ? undefined : outObj;
}
return (transform(data, '') as JsonObject) ?? {};
}
export async function loadSchema(options: Options): Promise<ConfigSchema> {
const start = process.hrtime();
const schemas = await collectConfigSchemas(options.dependencies[0]);
const [durS, durNs] = process.hrtime(start);
const dur = (durS + durNs / 10 ** 9).toFixed(3);
console.log(`DEBUG: collected config schemas in ${dur}s`);
console.log('DEBUG: schemas =', schemas);
const validate = compileConfigSchemas(schemas);
return {
process(
configs: AppConfig[],
{ visibilities }: ConfigProcessingOptions = {},
): AppConfig[] {
const result = validate(configs);
if (result.errors) {
throw new Error(
`Config validation failed, ${result.errors.join('; ')}`,
);
}
let processedConfigs = configs;
if (visibilities) {
processedConfigs = processedConfigs.map(({ data, context }) => ({
context,
data: filterByVisibility(data, visibilities, result.visibilityByPath),
}));
}
console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath);
return processedConfigs;
},
};
}
function compileConfigSchemas(
schemas: ConfigSchemaPackageEntry[],
): ValidationFunc {
const visibilityByPath = new Map<string, ConfigVisibility>();
const ajv = new Ajv({
strict: true,
allErrors: true,
defaultMeta: 'http://json-schema.org/draft-07/schema#',
schemas: {
'https://backstage.io/schema/config-v1': true,
},
keywords: [
{
keyword: 'visibility',
schemaType: 'string',
metaSchema: {
type: 'string',
enum: CONFIG_VISIBILITIES,
},
compile(visibility: ConfigVisibility) {
return (_data, ctx) => {
if (!ctx) {
return false;
}
if (visibility) {
visibilityByPath.set(ctx.dataPath, visibility);
}
return true;
};
},
},
],
});
const merged = mergeAllOf(
{ allOf: schemas.map(_ => _.value) },
{
ignoreAdditionalProperties: true,
resolvers: {
visibility(values: string[], path: string[]) {
const hasApp = values.some(_ => _ === 'frontend');
const hasSecret = values.some(_ => _ === 'secret');
if (hasApp && hasSecret) {
throw new Error(
`Config schema visibility is both 'frontend' and 'secret' for ${path.join(
'/',
)}`,
);
} else if (hasApp) {
return 'frontend';
} else if (hasSecret) {
return 'secret';
}
return 'backend';
},
} as Partial<Resolvers<JSONSchema>>,
},
);
const validate = ajv.compile(merged);
return configs => {
const config = ConfigReader.fromConfigs(configs).get();
visibilityByPath.clear();
const valid = validate(config);
if (!valid) {
const errors = ajv.errorsText(validate.errors);
return {
errors: [errors],
visibilityByPath: new Map(),
};
}
return {
visibilityByPath: new Map(visibilityByPath),
};
};
}
const req =
typeof __non_webpack_require__ === 'undefined'
? require
: __non_webpack_require__;
async function collectConfigSchemas(
name: string,
opts: unknown = {},
visited: Set<string> = new Set(),
schemas: ConfigSchemaPackageEntry[] = [],
): Promise<ConfigSchemaPackageEntry[]> {
const pkgPath = req.resolve(`${name}/package.json`, opts);
if (visited.has(pkgPath)) {
return schemas;
}
visited.add(pkgPath);
const pkg = await fs.readJson(pkgPath);
const depNames = [
...Object.keys(pkg.dependencies ?? {}),
...Object.keys(pkg.peerDependencies ?? {}),
];
// TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph,
// since that's pretty slow. We probably need a better way to determine when
// we've left the Backstage ecosystem, but this will do for now.
const hasSchema = 'configSchema' in pkg;
const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/'));
if (!hasSchema && !hasBackstageDep) {
return schemas;
}
if (hasSchema) {
if (typeof pkg.configSchema === 'string') {
if (!pkg.configSchema.endsWith('.json')) {
throw new Error(
`Config schema files must be .json, got ${pkg.configSchema}`,
);
}
const value = await fs.readJson(
resolvePath(dirname(pkgPath), pkg.configSchema),
);
schemas.push({
value,
path: pkgPath,
});
} else {
schemas.push({
value: pkg.configSchema,
path: pkgPath,
});
}
}
for (const depName of depNames) {
await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas);
}
return schemas;
}
@@ -0,0 +1,78 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { resolve as resolvePath, dirname } from 'path';
import { ConfigSchemaPackageEntry } from './types';
const req =
typeof __non_webpack_require__ === 'undefined'
? require
: __non_webpack_require__;
export async function collectConfigSchemas(
name: string,
opts: unknown = {},
visited: Set<string> = new Set(),
schemas: ConfigSchemaPackageEntry[] = [],
): Promise<ConfigSchemaPackageEntry[]> {
const pkgPath = req.resolve(`${name}/package.json`, opts);
if (visited.has(pkgPath)) {
return schemas;
}
visited.add(pkgPath);
const pkg = await fs.readJson(pkgPath);
const depNames = [
...Object.keys(pkg.dependencies ?? {}),
...Object.keys(pkg.peerDependencies ?? {}),
];
// TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph,
// since that's pretty slow. We probably need a better way to determine when
// we've left the Backstage ecosystem, but this will do for now.
const hasSchema = 'configSchema' in pkg;
const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/'));
if (!hasSchema && !hasBackstageDep) {
return schemas;
}
if (hasSchema) {
if (typeof pkg.configSchema === 'string') {
if (!pkg.configSchema.endsWith('.json')) {
throw new Error(
`Config schema files must be .json, got ${pkg.configSchema}`,
);
}
const value = await fs.readJson(
resolvePath(dirname(pkgPath), pkg.configSchema),
);
schemas.push({
value,
path: pkgPath,
});
} else {
schemas.push({
value: pkg.configSchema,
path: pkgPath,
});
}
}
for (const depName of depNames) {
await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas);
}
return schemas;
}
@@ -0,0 +1,109 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Ajv from 'ajv';
import { JSONSchema7 as JSONSchema } from 'json-schema';
import mergeAllOf, { Resolvers } from 'json-schema-merge-allof';
import { ConfigReader } from '@backstage/config';
import {
ConfigSchemaPackageEntry,
ValidationFunc,
CONFIG_VISIBILITIES,
ConfigVisibility,
} from './types';
export function compileConfigSchemas(
schemas: ConfigSchemaPackageEntry[],
): ValidationFunc {
const visibilityByPath = new Map<string, ConfigVisibility>();
const ajv = new Ajv({
strict: true,
allErrors: true,
defaultMeta: 'http://json-schema.org/draft-07/schema#',
schemas: {
'https://backstage.io/schema/config-v1': true,
},
keywords: [
{
keyword: 'visibility',
schemaType: 'string',
metaSchema: {
type: 'string',
enum: CONFIG_VISIBILITIES,
},
compile(visibility: ConfigVisibility) {
return (_data, ctx) => {
if (!ctx) {
return false;
}
if (visibility) {
visibilityByPath.set(ctx.dataPath, visibility);
}
return true;
};
},
},
],
});
const merged = mergeAllOf(
{ allOf: schemas.map(_ => _.value) },
{
ignoreAdditionalProperties: true,
resolvers: {
visibility(values: string[], path: string[]) {
const hasApp = values.some(_ => _ === 'frontend');
const hasSecret = values.some(_ => _ === 'secret');
if (hasApp && hasSecret) {
throw new Error(
`Config schema visibility is both 'frontend' and 'secret' for ${path.join(
'/',
)}`,
);
} else if (hasApp) {
return 'frontend';
} else if (hasSecret) {
return 'secret';
}
return 'backend';
},
} as Partial<Resolvers<JSONSchema>>,
},
);
const validate = ajv.compile(merged);
return configs => {
const config = ConfigReader.fromConfigs(configs).get();
visibilityByPath.clear();
const valid = validate(config);
if (!valid) {
const errors = ajv.errorsText(validate.errors);
return {
errors: [errors],
visibilityByPath: new Map(),
};
}
return {
visibilityByPath: new Map(visibilityByPath),
};
};
}
@@ -0,0 +1,64 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject, JsonValue } from '@backstage/config';
import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types';
export function filterByVisibility(
data: JsonObject,
includeVisibilities: ConfigVisibility[],
visibilityByPath: Map<string, ConfigVisibility>,
): JsonObject {
function transform(jsonVal: JsonValue, path: string): JsonValue | undefined {
if (typeof jsonVal !== 'object') {
const visibility =
visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY;
if (includeVisibilities.includes(visibility)) {
return jsonVal;
}
return undefined;
} else if (jsonVal === null) {
return undefined;
} else if (Array.isArray(jsonVal)) {
const arr = new Array<JsonValue>();
for (const [index, value] of jsonVal.entries()) {
const out = transform(value, `${path}/${index}`);
if (out !== undefined) {
arr.push(out);
}
}
return arr.length === 0 ? undefined : arr;
}
const outObj: JsonObject = {};
for (const [key, value] of Object.entries(jsonVal)) {
if (value === undefined) {
continue;
}
const out = transform(value, `${path}/${key}`);
if (out !== undefined) {
outObj[key] = out;
}
}
return Object.keys(outObj).length === 0 ? undefined : outObj;
}
return (transform(data, '') as JsonObject) ?? {};
}
@@ -0,0 +1,17 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { loadConfigSchema } from './load';
@@ -0,0 +1,63 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AppConfig } from '@backstage/config';
import { compileConfigSchemas } from './compile';
import { collectConfigSchemas } from './collect';
import { filterByVisibility } from './filtering';
import { ConfigSchema } from './types';
type Options = {
dependencies: string[];
};
export async function loadConfigSchema(
options: Options,
): Promise<ConfigSchema> {
const start = process.hrtime();
const schemas = await collectConfigSchemas(options.dependencies[0]);
const [durS, durNs] = process.hrtime(start);
const dur = (durS + durNs / 10 ** 9).toFixed(3);
console.log(`DEBUG: collected config schemas in ${dur}s`);
console.log('DEBUG: schemas =', schemas);
const validate = compileConfigSchemas(schemas);
return {
process(configs: AppConfig[], { visibilities } = {}): AppConfig[] {
const result = validate(configs);
if (result.errors) {
throw new Error(
`Config validation failed, ${result.errors.join('; ')}`,
);
}
let processedConfigs = configs;
if (visibilities) {
processedConfigs = processedConfigs.map(({ data, context }) => ({
context,
data: filterByVisibility(data, visibilities, result.visibilityByPath),
}));
}
console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath);
return processedConfigs;
},
};
}
@@ -0,0 +1,49 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AppConfig } from '@backstage/config';
import { JSONSchema7 as JSONSchema } from 'json-schema';
export type ConfigSchemaPackageEntry = {
value: JSONSchema;
path: string;
};
export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const;
export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number];
export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend';
type ValidationError = string;
type ValidationResult = {
errors?: ValidationError[];
visibilityByPath: Map<string, ConfigVisibility>;
};
export type ValidationFunc = (configs: AppConfig[]) => ValidationResult;
type ConfigProcessingOptions = {
visibilities?: ConfigVisibility[];
};
export type ConfigSchema = {
process(
appConfigs: AppConfig[],
options?: ConfigProcessingOptions,
): AppConfig[];
};
+2 -4
View File
@@ -17,7 +17,7 @@
import fs from 'fs-extra';
import { resolve as resolvePath, dirname, isAbsolute } from 'path';
import { AppConfig, JsonObject } from '@backstage/config';
import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib';
import { readConfigFile, readEnvConfig, readSecret } from './lib';
export type LoadConfigOptions = {
// The root directory of the config loading context. Used to find default configs.
@@ -80,8 +80,6 @@ export async function loadConfig(
const { configRoot } = options;
const configPaths = options.configPaths.slice();
const schema = await loadSchema({ dependencies: ['example-backend'] });
// If no paths are provided, we default to reading
// `app-config.yaml` and, if it exists, `app-config.local.yaml`
if (configPaths.length === 0) {
@@ -130,5 +128,5 @@ export async function loadConfig(
configs.push(...readEnvConfig(process.env));
return schema.process(configs, { visibilities: ['frontend', 'secret'] });
return configs;
}