From d4f0a1406061cb85d8d51c7cc6ec7a647faba55c Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Wed, 17 Feb 2021 12:14:07 +0100 Subject: [PATCH 1/7] backstage-cli: add config:schema command. Signed-off-by: Andreas Stenius --- .changeset/short-geese-double.md | 5 ++++ packages/cli/src/commands/config/schema.ts | 35 ++++++++++++++++++++++ packages/cli/src/commands/index.ts | 9 ++++++ packages/cli/src/lib/config.ts | 6 ++-- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 .changeset/short-geese-double.md create mode 100644 packages/cli/src/commands/config/schema.ts diff --git a/.changeset/short-geese-double.md b/.changeset/short-geese-double.md new file mode 100644 index 0000000000..45d3764e6d --- /dev/null +++ b/.changeset/short-geese-double.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': minor +--- + +New config command to export the configuration schema. When running backstage-cli with yarn, consider using `yarn --silent backstage-cli config:schema` to get a clean output on `stdout`. diff --git a/packages/cli/src/commands/config/schema.ts b/packages/cli/src/commands/config/schema.ts new file mode 100644 index 0000000000..1d6adf72d6 --- /dev/null +++ b/packages/cli/src/commands/config/schema.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Command } from 'commander'; +import { stringify as stringifyYaml } from 'yaml'; +import { loadCliConfig } from '../../lib/config'; + +export default async (cmd: Command) => { + const { schema } = await loadCliConfig({ + args: [], + fromPackage: cmd.package, + mockEnv: true, + }); + + const data = schema.serialize(); + + if (cmd.format === 'json') { + process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); + } else { + process.stdout.write(`${stringifyYaml(data)}\n`); + } +}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 1d267c6889..5fd7817a83 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -177,6 +177,15 @@ export function registerCommands(program: CommanderStatic) { ) .action(lazy(() => import('./config/validate').then(m => m.default))); + program + .command('config:schema') + .option( + '--package ', + 'Only output config schema that applies to the given package', + ) + .description('Print configuration schema') + .action(lazy(() => import('./config/schema').then(m => m.default))); + program .command('versions:bump') .description('Bump Backstage packages to the latest versions') diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index b49b644e25..de6bd6353d 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -48,8 +48,10 @@ export async function loadCliConfig(options: Options) { configPaths, }); - console.log( - `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, + // printing to stderr to not clobber stdout in case the cli command + // outputs structured data (e.g. as config:schema does) + process.stderr.write( + `Loaded config from ${appConfigs.map(c => c.context).join(', ')}\n`, ); try { From 4b01c7caaf134744d764ff3c27a6a04c8de6c7f8 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 19 Feb 2021 11:17:15 +0100 Subject: [PATCH 2/7] docs(cli): describe config:schema Signed-off-by: Andreas Stenius --- docs/cli/commands.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 726f2a922c..562604db97 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -47,6 +47,7 @@ remove-plugin Removes plugin in the current repository config:print Print the app configuration for the current package config:check Validate that the given configuration loads and matches schema +config:schema Dump the app configuration schema versions:bump Bump Backstage packages to the latest versions versions:check Check Backstage package versioning @@ -504,6 +505,27 @@ Options: -h, --help display help for command ``` +## config:schema + +Scope: `root` + +Dump the configuration schema that was collected from all local packages in the +repo. + +Note: when run by `yarn`, supply the yarn option `--silent` if you are using the +output in a command line pipe to avoid non schema output in the pipeline. + +```text +Usage: backstage-cli config:schema [options] + +Print configuration schema + +Options: + --package <name> Only output config schema that applies to the given package + -h, --help display help for command + +``` + ## versions:bump Scope: `root` From 0e0d0595ea7fea27b59a6546debd4a347c64dca3 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 19 Feb 2021 14:43:25 +0100 Subject: [PATCH 3/7] config(schema): export mergeConfigSchemas from config-loader for use by backstage-cli config:schema. Signed-off-by: Andreas Stenius --- packages/cli/src/commands/config/schema.ts | 3 +- packages/config-loader/src/index.ts | 2 +- .../config-loader/src/lib/schema/compile.ts | 66 ++++++++++++------- .../config-loader/src/lib/schema/index.ts | 1 + 4 files changed, 46 insertions(+), 26 deletions(-) diff --git a/packages/cli/src/commands/config/schema.ts b/packages/cli/src/commands/config/schema.ts index 1d6adf72d6..6badec6b6d 100644 --- a/packages/cli/src/commands/config/schema.ts +++ b/packages/cli/src/commands/config/schema.ts @@ -17,6 +17,7 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; import { loadCliConfig } from '../../lib/config'; +import { mergeConfigSchemas } from '@backstage/config-loader'; export default async (cmd: Command) => { const { schema } = await loadCliConfig({ @@ -25,7 +26,7 @@ export default async (cmd: Command) => { mockEnv: true, }); - const data = schema.serialize(); + const { schema: data } = mergeConfigSchemas(schema.serialize().schemas); if (cmd.format === 'json') { process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 9ad54c5f18..d9e5ae1350 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export { readEnvConfig, loadConfigSchema } from './lib'; +export { readEnvConfig, loadConfigSchema, mergeConfigSchemas } from './lib'; export type { ConfigSchema, ConfigVisibility } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index e340d775a1..0cbd075893 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -35,6 +35,44 @@ import { export function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { + const { schema: merged, parser: ajv, visibilityByPath } = mergeConfigSchemas( + schemas, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilityByPath.clear(); + + const valid = validate(config); + if (!valid) { + const errors = validate.errors ?? []; + return { + errors: errors.map(({ dataPath, message, params }) => { + const paramStr = Object.entries(params) + .map(([name, value]) => `${name}=${value}`) + .join(' '); + return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; + }), + visibilityByPath: new Map(), + }; + } + + return { + visibilityByPath: new Map(visibilityByPath), + }; + }; +} + +/** + * Given a list of configuration schemas from packages, merge them + * into a single json schema. + */ +export function mergeConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): { schema: JSONSchema; parser: Ajv } { // The ajv instance below is stateful and doesn't really allow for additional // output during validation. We work around this by having this extra piece // of state that we reset before each validation. @@ -108,29 +146,9 @@ export function compileConfigSchemas( }, ); - const validate = ajv.compile(merged); - - return configs => { - const config = ConfigReader.fromConfigs(configs).get(); - - visibilityByPath.clear(); - - const valid = validate(config); - if (!valid) { - const errors = validate.errors ?? []; - return { - errors: errors.map(({ dataPath, message, params }) => { - const paramStr = Object.entries(params) - .map(([name, value]) => `${name}=${value}`) - .join(' '); - return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; - }), - visibilityByPath: new Map(), - }; - } - - return { - visibilityByPath: new Map(visibilityByPath), - }; + return { + schema: merged, + parser: ajv, + visibilityByPath, }; } diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index 8cefb93b3c..00bb5c7d10 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -14,5 +14,6 @@ * limitations under the License. */ +export { mergeConfigSchemas } from './compile'; export { loadConfigSchema } from './load'; export type { ConfigSchema, ConfigVisibility } from './types'; From 64a98b5fe6016477ac3b4b379799495b9e449432 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Mon, 22 Feb 2021 10:40:51 +0100 Subject: [PATCH 4/7] cli: fix types. Signed-off-by: Andreas Stenius --- packages/cli/src/commands/config/schema.ts | 9 +++++++-- packages/config-loader/src/index.ts | 6 +++++- packages/config-loader/src/lib/schema/compile.ts | 6 +++++- packages/config-loader/src/lib/schema/index.ts | 6 +++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/commands/config/schema.ts b/packages/cli/src/commands/config/schema.ts index 6badec6b6d..17525ec48b 100644 --- a/packages/cli/src/commands/config/schema.ts +++ b/packages/cli/src/commands/config/schema.ts @@ -17,7 +17,10 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; import { loadCliConfig } from '../../lib/config'; -import { mergeConfigSchemas } from '@backstage/config-loader'; +import { + mergeConfigSchemas, + ConfigSchemaPackageEntry, +} from '@backstage/config-loader'; export default async (cmd: Command) => { const { schema } = await loadCliConfig({ @@ -26,7 +29,9 @@ export default async (cmd: Command) => { mockEnv: true, }); - const { schema: data } = mergeConfigSchemas(schema.serialize().schemas); + const { schema: data } = mergeConfigSchemas( + schema.serialize().schemas as ConfigSchemaPackageEntry[], + ); if (cmd.format === 'json') { process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index d9e5ae1350..4878e9d098 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,6 +15,10 @@ */ export { readEnvConfig, loadConfigSchema, mergeConfigSchemas } from './lib'; -export type { ConfigSchema, ConfigVisibility } from './lib'; +export type { + ConfigSchema, + ConfigSchemaPackageEntry, + ConfigVisibility, +} from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 0cbd075893..0ac1ea8045 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -72,7 +72,11 @@ export function compileConfigSchemas( */ export function mergeConfigSchemas( schemas: ConfigSchemaPackageEntry[], -): { schema: JSONSchema; parser: Ajv } { +): { + schema: JSONSchema; + parser: Ajv; + visibilityByPath: Map; +} { // The ajv instance below is stateful and doesn't really allow for additional // output during validation. We work around this by having this extra piece // of state that we reset before each validation. diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index 00bb5c7d10..dc2e471a28 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -16,4 +16,8 @@ export { mergeConfigSchemas } from './compile'; export { loadConfigSchema } from './load'; -export type { ConfigSchema, ConfigVisibility } from './types'; +export type { + ConfigSchema, + ConfigSchemaPackageEntry, + ConfigVisibility, +} from './types'; From fb30fc103bea1504bfbd65d724e2e2ac7f1a8bd6 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Wed, 24 Feb 2021 09:27:54 +0100 Subject: [PATCH 5/7] cli(config:schema): cleanups. Signed-off-by: Andreas Stenius --- .changeset/short-geese-double.md | 2 +- packages/cli/src/commands/config/schema.ts | 12 ++- packages/config-loader/src/index.ts | 6 +- .../config-loader/src/lib/schema/compile.ts | 84 ++++++++----------- .../config-loader/src/lib/schema/index.ts | 6 +- 5 files changed, 46 insertions(+), 64 deletions(-) diff --git a/.changeset/short-geese-double.md b/.changeset/short-geese-double.md index 45d3764e6d..1c63321be7 100644 --- a/.changeset/short-geese-double.md +++ b/.changeset/short-geese-double.md @@ -1,5 +1,5 @@ --- -'@backstage/cli': minor +'@backstage/cli': patch --- New config command to export the configuration schema. When running backstage-cli with yarn, consider using `yarn --silent backstage-cli config:schema` to get a clean output on `stdout`. diff --git a/packages/cli/src/commands/config/schema.ts b/packages/cli/src/commands/config/schema.ts index 17525ec48b..fbf46bff1a 100644 --- a/packages/cli/src/commands/config/schema.ts +++ b/packages/cli/src/commands/config/schema.ts @@ -29,13 +29,17 @@ export default async (cmd: Command) => { mockEnv: true, }); - const { schema: data } = mergeConfigSchemas( - schema.serialize().schemas as ConfigSchemaPackageEntry[], + const merged = mergeConfigSchemas( + schema.serialize().schemas.map(_ => _.value), ); + merged.title = 'Application Configuration Schema'; + merged.description = + 'This is the schema describing the structure of the app-config.yaml configuration file.'; + if (cmd.format === 'json') { - process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(merged, null, 2)}\n`); } else { - process.stdout.write(`${stringifyYaml(data)}\n`); + process.stdout.write(`${stringifyYaml(merged)}\n`); } }; diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 4878e9d098..d9e5ae1350 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,10 +15,6 @@ */ export { readEnvConfig, loadConfigSchema, mergeConfigSchemas } from './lib'; -export type { - ConfigSchema, - ConfigSchemaPackageEntry, - ConfigVisibility, -} from './lib'; +export type { ConfigSchema, ConfigVisibility } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 0ac1ea8045..e85b6023cc 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -35,48 +35,6 @@ import { export function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { - const { schema: merged, parser: ajv, visibilityByPath } = mergeConfigSchemas( - schemas, - ); - - const validate = ajv.compile(merged); - - return configs => { - const config = ConfigReader.fromConfigs(configs).get(); - - visibilityByPath.clear(); - - const valid = validate(config); - if (!valid) { - const errors = validate.errors ?? []; - return { - errors: errors.map(({ dataPath, message, params }) => { - const paramStr = Object.entries(params) - .map(([name, value]) => `${name}=${value}`) - .join(' '); - return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; - }), - visibilityByPath: new Map(), - }; - } - - return { - visibilityByPath: new Map(visibilityByPath), - }; - }; -} - -/** - * Given a list of configuration schemas from packages, merge them - * into a single json schema. - */ -export function mergeConfigSchemas( - schemas: ConfigSchemaPackageEntry[], -): { - schema: JSONSchema; - parser: Ajv; - visibilityByPath: Map; -} { // The ajv instance below is stateful and doesn't really allow for additional // output during validation. We work around this by having this extra piece // of state that we reset before each validation. @@ -119,8 +77,41 @@ export function mergeConfigSchemas( } } + const merged = mergeConfigSchemas(schemas.map(_ => _.value)); + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilityByPath.clear(); + + const valid = validate(config); + if (!valid) { + const errors = validate.errors ?? []; + return { + errors: errors.map(({ dataPath, message, params }) => { + const paramStr = Object.entries(params) + .map(([name, value]) => `${name}=${value}`) + .join(' '); + return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; + }), + visibilityByPath: new Map(), + }; + } + + return { + visibilityByPath: new Map(visibilityByPath), + }; + }; +} + +/** + * Given a list of configuration schemas from packages, merge them + * into a single json schema. + */ +export function mergeConfigSchemas(schemas: JSONSchema[]): JSONSchema { const merged = mergeAllOf( - { allOf: schemas.map(_ => _.value) }, + { allOf: schemas }, { // JSONSchema is typically subtractive, as in it always reduces the set of allowed // inputs through constraints. This changes the object property merging to be additive @@ -149,10 +140,5 @@ export function mergeConfigSchemas( } as Partial>, }, ); - - return { - schema: merged, - parser: ajv, - visibilityByPath, - }; + return merged; } diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index dc2e471a28..00bb5c7d10 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -16,8 +16,4 @@ export { mergeConfigSchemas } from './compile'; export { loadConfigSchema } from './load'; -export type { - ConfigSchema, - ConfigSchemaPackageEntry, - ConfigVisibility, -} from './types'; +export type { ConfigSchema, ConfigVisibility } from './types'; From bddf102f0fba5fb4601b7eb376a44fe5fca8226c Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Wed, 24 Feb 2021 09:40:47 +0100 Subject: [PATCH 6/7] cli: type fixes. Signed-off-by: Andreas Stenius --- packages/cli/package.json | 1 + packages/cli/src/commands/config/schema.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 4a95df0f38..47d4c73cc9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -82,6 +82,7 @@ "inquirer": "^7.0.4", "jest": "^26.0.1", "jest-css-modules": "^2.1.0", + "json-schema": "^0.2.5", "lodash": "^4.17.19", "mini-css-extract-plugin": "^0.9.0", "ora": "^4.0.3", diff --git a/packages/cli/src/commands/config/schema.ts b/packages/cli/src/commands/config/schema.ts index fbf46bff1a..63c1524789 100644 --- a/packages/cli/src/commands/config/schema.ts +++ b/packages/cli/src/commands/config/schema.ts @@ -15,12 +15,11 @@ */ import { Command } from 'commander'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; import { stringify as stringifyYaml } from 'yaml'; import { loadCliConfig } from '../../lib/config'; -import { - mergeConfigSchemas, - ConfigSchemaPackageEntry, -} from '@backstage/config-loader'; +import { JsonObject } from '@backstage/config'; +import { mergeConfigSchemas } from '@backstage/config-loader'; export default async (cmd: Command) => { const { schema } = await loadCliConfig({ @@ -30,7 +29,9 @@ export default async (cmd: Command) => { }); const merged = mergeConfigSchemas( - schema.serialize().schemas.map(_ => _.value), + (schema.serialize().schemas as JsonObject[]).map( + _ => _.value as JSONSchema, + ), ); merged.title = 'Application Configuration Schema'; From 46dab9e09c35a6bf179d60d698a061936a73fe48 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 26 Feb 2021 12:02:36 +0100 Subject: [PATCH 7/7] config-loader: move @types/json-schema to dependencies. Signed-off-by: Andreas Stenius --- packages/config-loader/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index b905d3e8a4..07fd22d7ba 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -32,6 +32,7 @@ "dependencies": { "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", + "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", "fs-extra": "^9.0.0", "json-schema": "^0.2.5", @@ -42,7 +43,6 @@ }, "devDependencies": { "@types/jest": "^26.0.7", - "@types/json-schema": "^7.0.6", "@types/json-schema-merge-allof": "^0.6.0", "@types/mock-fs": "^4.10.0", "@types/node": "^12.0.0",