From 8c65cec932d2818914b0ef124059e832016952a9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:18:58 +0100 Subject: [PATCH] config-loader: split up config schema implementation and move into separate lib --- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/lib/index.ts | 2 +- packages/config-loader/src/lib/schema.ts | 289 ------------------ .../config-loader/src/lib/schema/collect.ts | 78 +++++ .../config-loader/src/lib/schema/compile.ts | 109 +++++++ .../config-loader/src/lib/schema/filtering.ts | 64 ++++ .../config-loader/src/lib/schema/index.ts | 17 ++ packages/config-loader/src/lib/schema/load.ts | 63 ++++ .../config-loader/src/lib/schema/types.ts | 49 +++ packages/config-loader/src/loader.ts | 6 +- 10 files changed, 384 insertions(+), 295 deletions(-) delete mode 100644 packages/config-loader/src/lib/schema.ts create mode 100644 packages/config-loader/src/lib/schema/collect.ts create mode 100644 packages/config-loader/src/lib/schema/compile.ts create mode 100644 packages/config-loader/src/lib/schema/filtering.ts create mode 100644 packages/config-loader/src/lib/schema/index.ts create mode 100644 packages/config-loader/src/lib/schema/load.ts create mode 100644 packages/config-loader/src/lib/schema/types.ts diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 02db134fe5..415b6d4022 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export { readEnvConfig } from './lib'; +export { readEnvConfig, loadConfigSchema } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 29812053bc..4da0c97788 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,4 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; -export { loadSchema } from './schema'; +export { loadConfigSchema } from './schema'; diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts deleted file mode 100644 index 0d820ab173..0000000000 --- a/packages/config-loader/src/lib/schema.ts +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { resolve as resolvePath, dirname } from 'path'; -import Ajv from 'ajv'; -import { JSONSchema7 as JSONSchema } from 'json-schema'; -import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; -import { - AppConfig, - ConfigReader, - JsonObject, - JsonValue, -} from '@backstage/config'; - -const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; - -type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; - -const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; - -type ConfigSchema = { - process( - appConfigs: AppConfig[], - options?: ConfigProcessingOptions, - ): AppConfig[]; -}; - -type ConfigSchemaPackageEntry = { - value: JSONSchema; - path: string; -}; - -type Options = { - dependencies: string[]; -}; - -type ValidationError = string; - -type ValidationResult = { - errors?: ValidationError[]; - visibilityByPath: Map; -}; - -type ValidationFunc = (configs: AppConfig[]) => ValidationResult; - -type ConfigProcessingOptions = { - visibilities?: ConfigVisibility[]; -}; - -export function filterByVisibility( - data: JsonObject, - includeVisibilities: ConfigVisibility[], - visibilityByPath: Map, -): JsonObject { - function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - if (typeof jsonVal !== 'object') { - const visibility = - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; - if (includeVisibilities.includes(visibility)) { - return jsonVal; - } - return undefined; - } else if (jsonVal === null) { - return undefined; - } else if (Array.isArray(jsonVal)) { - const arr = new Array(); - - for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); - if (out !== undefined) { - arr.push(out); - } - } - - return arr.length === 0 ? undefined : arr; - } - - const outObj: JsonObject = {}; - - for (const [key, value] of Object.entries(jsonVal)) { - if (value === undefined) { - continue; - } - const out = transform(value, `${path}/${key}`); - if (out !== undefined) { - outObj[key] = out; - } - } - - return Object.keys(outObj).length === 0 ? undefined : outObj; - } - - return (transform(data, '') as JsonObject) ?? {}; -} - -export async function loadSchema(options: Options): Promise { - const start = process.hrtime(); - - const schemas = await collectConfigSchemas(options.dependencies[0]); - - const [durS, durNs] = process.hrtime(start); - const dur = (durS + durNs / 10 ** 9).toFixed(3); - console.log(`DEBUG: collected config schemas in ${dur}s`); - console.log('DEBUG: schemas =', schemas); - - const validate = compileConfigSchemas(schemas); - - return { - process( - configs: AppConfig[], - { visibilities }: ConfigProcessingOptions = {}, - ): AppConfig[] { - const result = validate(configs); - if (result.errors) { - throw new Error( - `Config validation failed, ${result.errors.join('; ')}`, - ); - } - - let processedConfigs = configs; - - if (visibilities) { - processedConfigs = processedConfigs.map(({ data, context }) => ({ - context, - data: filterByVisibility(data, visibilities, result.visibilityByPath), - })); - } - console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); - - return processedConfigs; - }, - }; -} - -function compileConfigSchemas( - schemas: ConfigSchemaPackageEntry[], -): ValidationFunc { - const visibilityByPath = new Map(); - - const ajv = new Ajv({ - strict: true, - allErrors: true, - defaultMeta: 'http://json-schema.org/draft-07/schema#', - schemas: { - 'https://backstage.io/schema/config-v1': true, - }, - keywords: [ - { - keyword: 'visibility', - schemaType: 'string', - metaSchema: { - type: 'string', - enum: CONFIG_VISIBILITIES, - }, - compile(visibility: ConfigVisibility) { - return (_data, ctx) => { - if (!ctx) { - return false; - } - if (visibility) { - visibilityByPath.set(ctx.dataPath, visibility); - } - return true; - }; - }, - }, - ], - }); - - const merged = mergeAllOf( - { allOf: schemas.map(_ => _.value) }, - { - ignoreAdditionalProperties: true, - resolvers: { - visibility(values: string[], path: string[]) { - const hasApp = values.some(_ => _ === 'frontend'); - const hasSecret = values.some(_ => _ === 'secret'); - if (hasApp && hasSecret) { - throw new Error( - `Config schema visibility is both 'frontend' and 'secret' for ${path.join( - '/', - )}`, - ); - } else if (hasApp) { - return 'frontend'; - } else if (hasSecret) { - return 'secret'; - } - - return 'backend'; - }, - } as Partial>, - }, - ); - - const validate = ajv.compile(merged); - - return configs => { - const config = ConfigReader.fromConfigs(configs).get(); - - visibilityByPath.clear(); - - const valid = validate(config); - if (!valid) { - const errors = ajv.errorsText(validate.errors); - return { - errors: [errors], - visibilityByPath: new Map(), - }; - } - - return { - visibilityByPath: new Map(visibilityByPath), - }; - }; -} - -const req = - typeof __non_webpack_require__ === 'undefined' - ? require - : __non_webpack_require__; - -async function collectConfigSchemas( - name: string, - opts: unknown = {}, - visited: Set = new Set(), - schemas: ConfigSchemaPackageEntry[] = [], -): Promise { - const pkgPath = req.resolve(`${name}/package.json`, opts); - if (visited.has(pkgPath)) { - return schemas; - } - visited.add(pkgPath); - const pkg = await fs.readJson(pkgPath); - const depNames = [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}), - ]; - - // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, - // since that's pretty slow. We probably need a better way to determine when - // we've left the Backstage ecosystem, but this will do for now. - const hasSchema = 'configSchema' in pkg; - const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); - if (!hasSchema && !hasBackstageDep) { - return schemas; - } - if (hasSchema) { - if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { - throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, - ); - } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: pkgPath, - }); - } else { - schemas.push({ - value: pkg.configSchema, - path: pkgPath, - }); - } - } - - for (const depName of depNames) { - await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); - } - - return schemas; -} diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts new file mode 100644 index 0000000000..b6d0bffaef --- /dev/null +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; +import { ConfigSchemaPackageEntry } from './types'; + +const req = + typeof __non_webpack_require__ === 'undefined' + ? require + : __non_webpack_require__; + +export async function collectConfigSchemas( + name: string, + opts: unknown = {}, + visited: Set = new Set(), + schemas: ConfigSchemaPackageEntry[] = [], +): Promise { + const pkgPath = req.resolve(`${name}/package.json`, opts); + if (visited.has(pkgPath)) { + return schemas; + } + visited.add(pkgPath); + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return schemas; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + for (const depName of depNames) { + await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); + } + + return schemas; +} diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts new file mode 100644 index 0000000000..9ab8627fce --- /dev/null +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ajv from 'ajv'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; +import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; +import { ConfigReader } from '@backstage/config'; +import { + ConfigSchemaPackageEntry, + ValidationFunc, + CONFIG_VISIBILITIES, + ConfigVisibility, +} from './types'; + +export function compileConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): ValidationFunc { + const visibilityByPath = new Map(); + + const ajv = new Ajv({ + strict: true, + allErrors: true, + defaultMeta: 'http://json-schema.org/draft-07/schema#', + schemas: { + 'https://backstage.io/schema/config-v1': true, + }, + keywords: [ + { + keyword: 'visibility', + schemaType: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, ctx) => { + if (!ctx) { + return false; + } + if (visibility) { + visibilityByPath.set(ctx.dataPath, visibility); + } + return true; + }; + }, + }, + ], + }); + + const merged = mergeAllOf( + { allOf: schemas.map(_ => _.value) }, + { + ignoreAdditionalProperties: true, + resolvers: { + visibility(values: string[], path: string[]) { + const hasApp = values.some(_ => _ === 'frontend'); + const hasSecret = values.some(_ => _ === 'secret'); + if (hasApp && hasSecret) { + throw new Error( + `Config schema visibility is both 'frontend' and 'secret' for ${path.join( + '/', + )}`, + ); + } else if (hasApp) { + return 'frontend'; + } else if (hasSecret) { + return 'secret'; + } + + return 'backend'; + }, + } as Partial>, + }, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilityByPath.clear(); + + const valid = validate(config); + if (!valid) { + const errors = ajv.errorsText(validate.errors); + return { + errors: [errors], + visibilityByPath: new Map(), + }; + } + + return { + visibilityByPath: new Map(visibilityByPath), + }; + }; +} diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts new file mode 100644 index 0000000000..64ccfa5e24 --- /dev/null +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, JsonValue } from '@backstage/config'; +import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; + +export function filterByVisibility( + data: JsonObject, + includeVisibilities: ConfigVisibility[], + visibilityByPath: Map, +): JsonObject { + function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + if (typeof jsonVal !== 'object') { + const visibility = + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + if (includeVisibilities.includes(visibility)) { + return jsonVal; + } + return undefined; + } else if (jsonVal === null) { + return undefined; + } else if (Array.isArray(jsonVal)) { + const arr = new Array(); + + for (const [index, value] of jsonVal.entries()) { + const out = transform(value, `${path}/${index}`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr.length === 0 ? undefined : arr; + } + + const outObj: JsonObject = {}; + + for (const [key, value] of Object.entries(jsonVal)) { + if (value === undefined) { + continue; + } + const out = transform(value, `${path}/${key}`); + if (out !== undefined) { + outObj[key] = out; + } + } + + return Object.keys(outObj).length === 0 ? undefined : outObj; + } + + return (transform(data, '') as JsonObject) ?? {}; +} diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts new file mode 100644 index 0000000000..990c0dc182 --- /dev/null +++ b/packages/config-loader/src/lib/schema/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { loadConfigSchema } from './load'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts new file mode 100644 index 0000000000..75fcb86ba7 --- /dev/null +++ b/packages/config-loader/src/lib/schema/load.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { compileConfigSchemas } from './compile'; +import { collectConfigSchemas } from './collect'; +import { filterByVisibility } from './filtering'; +import { ConfigSchema } from './types'; + +type Options = { + dependencies: string[]; +}; + +export async function loadConfigSchema( + options: Options, +): Promise { + const start = process.hrtime(); + + const schemas = await collectConfigSchemas(options.dependencies[0]); + + const [durS, durNs] = process.hrtime(start); + const dur = (durS + durNs / 10 ** 9).toFixed(3); + console.log(`DEBUG: collected config schemas in ${dur}s`); + console.log('DEBUG: schemas =', schemas); + + const validate = compileConfigSchemas(schemas); + + return { + process(configs: AppConfig[], { visibilities } = {}): AppConfig[] { + const result = validate(configs); + if (result.errors) { + throw new Error( + `Config validation failed, ${result.errors.join('; ')}`, + ); + } + + let processedConfigs = configs; + + if (visibilities) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility(data, visibilities, result.visibilityByPath), + })); + } + console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); + + return processedConfigs; + }, + }; +} diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts new file mode 100644 index 0000000000..9e5624ca6a --- /dev/null +++ b/packages/config-loader/src/lib/schema/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; + +export type ConfigSchemaPackageEntry = { + value: JSONSchema; + path: string; +}; + +export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; + +export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; + +export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; + +type ValidationError = string; + +type ValidationResult = { + errors?: ValidationError[]; + visibilityByPath: Map; +}; + +export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; + +type ConfigProcessingOptions = { + visibilities?: ConfigVisibility[]; +}; + +export type ConfigSchema = { + process( + appConfigs: AppConfig[], + options?: ConfigProcessingOptions, + ): AppConfig[]; +}; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2d3c2afd9c..00414f86eb 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; import { AppConfig, JsonObject } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib'; +import { readConfigFile, readEnvConfig, readSecret } from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -80,8 +80,6 @@ export async function loadConfig( const { configRoot } = options; const configPaths = options.configPaths.slice(); - const schema = await loadSchema({ dependencies: ['example-backend'] }); - // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` if (configPaths.length === 0) { @@ -130,5 +128,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return schema.process(configs, { visibilities: ['frontend', 'secret'] }); + return configs; }