diff --git a/.changeset/fresh-lions-compete.md b/.changeset/fresh-lions-compete.md index 906ed37665..76c2161c37 100644 --- a/.changeset/fresh-lions-compete.md +++ b/.changeset/fresh-lions-compete.md @@ -1,5 +1,5 @@ --- -'@backstage/cli': minor +'@backstage/repo-tools': minor --- Adding two new commands to support OpenAPI spec writing, `schema:openapi:generate` to generate the Typescript file that `@backstage/backend-openapi-utils` needs for typing and `schema:openapi:verify` to verify that this file exists and matches your `openapi.yaml` file. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7fb8c3d74..3788687ab2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: run: node scripts/verify-api-reference.js - name: verify openapi.yaml matches schema/openapi.ts files - run: yarn backstage-cli repo schema:openapi:verify + run: yarn backstage-repo-tools schema:openapi:verify - name: verify doc links run: node scripts/verify-links.js diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index fe536d4a23..bad849d283 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -209,8 +209,6 @@ Commands: clean prepack postpack - schema:openapi:verify - schema:openapi:generate help [command] ``` @@ -267,24 +265,6 @@ Options: -h, --help ``` -### `backstage-cli package schema:openapi:generate` - -``` -Usage: backstage-cli package schema:openapi:generate [options] - -Options: - -h, --help -``` - -### `backstage-cli package schema:openapi:verify` - -``` -Usage: backstage-cli package schema:openapi:verify [options] - -Options: - -h, --help -``` - ### `backstage-cli package start` ``` @@ -423,8 +403,6 @@ Commands: lint [options] clean list-deprecations [options] - schema:openapi:verify - schema:openapi:generate test [options] help [command] ``` @@ -471,24 +449,6 @@ Options: -h, --help ``` -### `backstage-cli repo schema:openapi:generate` - -``` -Usage: backstage-cli repo schema:openapi:generate [options] - -Options: - -h, --help -``` - -### `backstage-cli repo schema:openapi:verify` - -``` -Usage: backstage-cli repo schema:openapi:verify [options] - -Options: - -h, --help -``` - ### `backstage-cli repo test` ``` diff --git a/packages/cli/package.json b/packages/cli/package.json index cc1d0a4d1c..5974116843 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -30,7 +30,6 @@ "backstage-cli": "bin/backstage-cli" }, "dependencies": { - "@apidevtools/swagger-parser": "^10.1.0", "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", "@backstage/config-loader": "workspace:^", @@ -97,7 +96,6 @@ "jest-css-modules": "^2.1.0", "jest-environment-jsdom": "^29.0.2", "jest-runtime": "^29.0.2", - "js-yaml": "^4.1.0", "json-schema": "^0.4.0", "lodash": "^4.17.21", "mini-css-extract-plugin": "^2.4.2", @@ -106,7 +104,6 @@ "node-libs-browser": "^2.2.1", "npm-packlist": "^5.0.0", "ora": "^5.3.0", - "p-limit": "^3.0.2", "postcss": "^8.1.0", "process": "^0.11.10", "react-dev-utils": "^12.0.0-next.60", diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 713863b685..ba11a53cfd 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -80,20 +80,6 @@ export function registerRepoCommand(program: Command) { lazy(() => import('./repo/list-deprecations').then(m => m.command)), ); - command - .command('schema:openapi:verify') - .description( - 'Verify that all OpenAPI schemas are valid and have a matching `schemas/openapi.ts` file.', - ) - .action(lazy(() => import('./openapi/verify').then(m => m.bulkCommand))); - - command - .command('schema:openapi:generate') - .description( - 'Generates a Typescript file from an OpenAPI yaml spec. For use with the `@backstage/backend-openapi-utils` ApiRouter type.', - ) - .action(lazy(() => import('./openapi/generate').then(m => m.bulkCommand))); - command .command('test') .allowUnknownOption(true) // Allows the command to run, but we still need to parse raw args @@ -193,20 +179,6 @@ export function registerScriptCommand(program: Command) { .command('postpack') .description('Restores the changes made by the prepack command') .action(lazy(() => import('./pack').then(m => m.post))); - - command - .command('schema:openapi:verify') - .description( - 'Verifies that an OpenAPI spec is defined in both an `openapi.yaml` file and a `schema/openapi.ts` file.', - ) - .action(lazy(() => import('./openapi/verify').then(m => m.command))); - - command - .command('schema:openapi:generate') - .description( - 'Generates a `schema/openapi.ts` file from an OpenAPI spec defined in `openapi.yaml`.', - ) - .action(lazy(() => import('./openapi/generate').then(m => m.command))); } export function registerMigrateCommand(program: Command) { diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index 0e1b2d2a4c..857c705006 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -14,6 +14,8 @@ Options: Commands: api-reports [options] [paths...] type-deps + schema:openapi:verify [paths...] + schema:openapi:generate [paths...] help [command] ``` @@ -33,6 +35,24 @@ Options: -h, --help ``` +### `backstage-repo-tools schema:openapi:generate` + +``` +Usage: backstage-repo-tools schema:openapi:generate [options] [paths...] + +Options: + -h, --help +``` + +### `backstage-repo-tools schema:openapi:verify` + +``` +Usage: backstage-repo-tools schema:openapi:verify [options] [paths...] + +Options: + -h, --help +``` + ### `backstage-repo-tools type-deps` ``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 4a53cc9976..d5817ef5af 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -30,6 +30,7 @@ "backstage-repo-tools": "bin/backstage-repo-tools" }, "dependencies": { + "@apidevtools/swagger-parser": "^10.1.0", "@backstage/cli-common": "workspace:^", "@backstage/errors": "workspace:^", "@manypkg/get-packages": "^1.1.3", @@ -40,7 +41,10 @@ "fs-extra": "10.1.0", "glob": "^8.0.3", "is-glob": "^4.0.3", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21", "minimatch": "^5.1.1", + "p-limit": "^3.0.2", "ts-node": "^10.0.0" }, "devDependencies": { diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index a917975750..b673689b5a 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -15,7 +15,6 @@ */ import { OptionValues } from 'commander'; -import fs from 'fs-extra'; import { createTemporaryTsConfig, categorizePackageDirs, @@ -23,7 +22,7 @@ import { runCliExtraction, buildDocs, } from './api-extractor'; -import { findPackageDirs, paths as cliPaths } from '../../lib/paths'; +import { getMatchingWorkspacePaths, paths as cliPaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; type Options = { @@ -49,10 +48,7 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { const omitMessages = parseArrayOption(opts.omitMessages); const isAllPackages = !paths?.length; - const selectedPaths = isAllPackages - ? await getWorkspacePackagePathPatterns() - : paths; - const selectedPackageDirs = await findPackageDirs(selectedPaths); + const selectedPackageDirs = await getMatchingWorkspacePaths(paths); if (isAllPackages && !isCiBuild && !isDocsBuild) { console.log(''); @@ -111,26 +107,6 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { } }; -/** - * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root. - * - * If the file does not exist, or the "workspaces" field is not present, returns `undefined`. - * - * @returns {Promise} The list of package names, or `undefined` if not found. - */ -async function getWorkspacePackagePathPatterns() { - const pkgJson = await fs - .readJson(cliPaths.resolveTargetRoot('package.json')) - .catch(error => { - if (error.code === 'ENOENT') { - return undefined; - } - throw error; - }); - const workspaces = pkgJson?.workspaces?.packages; - return workspaces; -} - /** * Splits the input string on comma, and returns an array of the resulting substrings. * for `undefined` or an empty string, returns an empty array. diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index f283e9c743..7aa7acef0b 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -52,6 +52,20 @@ export function registerCommands(program: Command) { .command('type-deps') .description('Find inconsistencies in types of all packages and plugins') .action(lazy(() => import('./type-deps/type-deps').then(m => m.default))); + + program + .command('schema:openapi:verify [paths...]') + .description( + 'Verify that all OpenAPI schemas are valid and have a matching `schemas/openapi.ts` file.', + ) + .action(lazy(() => import('./openapi/verify').then(m => m.bulkCommand))); + + program + .command('schema:openapi:generate [paths...]') + .description( + 'Generates a Typescript file from an OpenAPI yaml spec. For use with the `@backstage/backend-openapi-utils` ApiRouter type.', + ) + .action(lazy(() => import('./openapi/generate').then(m => m.bulkCommand))); } // Wraps an action function so that it always exits and handles errors diff --git a/packages/cli/src/commands/openapi/generate.ts b/packages/repo-tools/src/commands/openapi/generate.ts similarity index 64% rename from packages/cli/src/commands/openapi/generate.ts rename to packages/repo-tools/src/commands/openapi/generate.ts index 14a1217b7b..8ce88c1dcc 100644 --- a/packages/cli/src/commands/openapi/generate.ts +++ b/packages/repo-tools/src/commands/openapi/generate.ts @@ -15,13 +15,10 @@ */ import fs from 'fs-extra'; -import { paths } from '../../lib/paths'; import YAML from 'js-yaml'; import chalk from 'chalk'; import { resolve } from 'path'; -import { PackageGraph } from '../../lib/monorepo'; -import pLimit from 'p-limit'; -import { relative as relativePath } from 'path'; +import { runner } from './runner'; async function generate( directoryPath: string, @@ -44,36 +41,9 @@ async function generate( ); } -export async function command() { - try { - await generate(paths.resolveTarget('.')); - console.log(chalk.green('OpenAPI files successfully generated.')); - } catch (err) { - console.error(chalk.red(err.message)); - process.exit(1); - } -} - -export async function bulkCommand(): Promise { - const packages = await PackageGraph.listTargetPackages(); - const limit = pLimit(5); - - const resultsList = await Promise.all( - packages.map(pkg => - limit(async () => { - let resultText = ''; - try { - await generate(pkg.dir, { skipMissingYamlFile: true }); - } catch (err) { - resultText = err.message; - } - - return { - relativeDir: relativePath(paths.targetRoot, pkg.dir), - resultText, - }; - }), - ), +export async function bulkCommand(paths: string[] = []): Promise { + const resultsList = await runner(paths, (dir: string) => + generate(dir, { skipMissingYamlFile: true }), ); let failed = false; @@ -93,5 +63,7 @@ export async function bulkCommand(): Promise { if (failed) { process.exit(1); + } else { + console.log(chalk.green('Generated all files.')); } } diff --git a/packages/repo-tools/src/commands/openapi/runner.ts b/packages/repo-tools/src/commands/openapi/runner.ts new file mode 100644 index 0000000000..4b1b0167bf --- /dev/null +++ b/packages/repo-tools/src/commands/openapi/runner.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getMatchingWorkspacePaths } from '../../lib/paths'; +import pLimit from 'p-limit'; +import { relative as relativePath } from 'path'; +import { paths as cliPaths } from '../../lib/paths'; + +export async function runner( + paths: string[], + command: (dir: string) => Promise, +) { + const packages = await getMatchingWorkspacePaths(paths); + const limit = pLimit(5); + + const resultsList = await Promise.all( + packages.map(pkg => + limit(async () => { + let resultText = ''; + try { + console.log(`## Processing ${pkg}`); + await command(pkg); + } catch (err) { + resultText = err.message; + } + + return { + relativeDir: relativePath(cliPaths.targetRoot, pkg), + resultText, + }; + }), + ), + ); + + return resultsList; +} diff --git a/packages/cli/src/commands/openapi/verify.ts b/packages/repo-tools/src/commands/openapi/verify.ts similarity index 51% rename from packages/cli/src/commands/openapi/verify.ts rename to packages/repo-tools/src/commands/openapi/verify.ts index ad0031622f..c45e388a07 100644 --- a/packages/cli/src/commands/openapi/verify.ts +++ b/packages/repo-tools/src/commands/openapi/verify.ts @@ -15,41 +15,16 @@ */ import fs from 'fs-extra'; -import { paths } from '../../lib/paths'; import YAML from 'js-yaml'; -import { isEqual } from 'lodash'; -import { join, resolve } from 'path'; +import { isEqual, cloneDeep } from 'lodash'; +import { join } from 'path'; import chalk from 'chalk'; -import { relative as relativePath } from 'path'; -import { PackageGraph } from '../../lib/monorepo'; -import { cloneDeep } from 'lodash'; +import { relative as relativePath, resolve as resolvePath } from 'path'; import Parser from '@apidevtools/swagger-parser'; -import { detectRoleFromPackage } from '../../lib/role'; -import pLimit from 'p-limit'; - -const SUPPORTED_ROLES = [ - 'backend', - 'backend-plugin', - 'backend-plugin-module', - 'node-library', -]; - -async function verify( - directoryPath: string, - config: { checkRole: boolean } = { checkRole: false }, -) { - const { checkRole } = config ?? {}; - if (checkRole) { - const role = detectRoleFromPackage( - await fs.readJson(resolve(directoryPath, 'package.json')), - ); - - if (!role || !SUPPORTED_ROLES.includes(role)) { - console.log(chalk.red(`Unsupported role ${role}`)); - process.exit(1); - } - } +import { runner } from './runner'; +import { paths as cliPaths } from '../../lib/paths'; +async function verify(directoryPath: string) { const openapiPath = join(directoryPath, 'openapi.yaml'); if (!(await fs.pathExists(openapiPath))) { return; @@ -62,51 +37,27 @@ async function verify( if (!(await fs.pathExists(schemaPath))) { throw new Error('No `schema/openapi.ts` file found.'); } - const schema = await import(join(directoryPath, 'schema/openapi')); + + const schema = await import( + resolvePath(join(directoryPath, 'schema/openapi')) + ); + if (!schema.default) { throw new Error('`schemas/openapi.ts` needs to have a default export.'); } if (!isEqual(schema.default, yaml)) { throw new Error( `\`openapi.yaml\` and \`schema/openapi.ts\` do not match. Please run \`yarn --cwd ${relativePath( - paths.targetRoot, + cliPaths.targetRoot, directoryPath, )} schema:openapi:generate\` to regenerate \`schema/openapi.ts\`.`, ); } } -export async function command() { - try { - await verify(paths.resolveTarget('.'), { checkRole: true }); - console.log(chalk.green('OpenAPI files are valid.')); - } catch (err) { - console.error(chalk.red(err.message)); - process.exit(1); - } -} - -export async function bulkCommand(): Promise { - const packages = await PackageGraph.listTargetPackages(); - const limit = pLimit(5); - - const resultsList = await Promise.all( - packages.map(pkg => - limit(async () => { - let resultText = ''; - try { - await verify(pkg.dir); - } catch (err) { - resultText = err.message; - } - - return { - relativeDir: relativePath(paths.targetRoot, pkg.dir), - resultText, - }; - }), - ), - ); +export async function bulkCommand(paths: string[] = []): Promise { + console.log(paths); + const resultsList = await runner(paths, dir => verify(dir)); let failed = false; for (const { relativeDir, resultText } of resultsList) { @@ -121,5 +72,7 @@ export async function bulkCommand(): Promise { if (failed) { process.exit(1); + } else { + console.log(chalk.green('Verified all files.')); } } diff --git a/packages/repo-tools/src/lib/paths.ts b/packages/repo-tools/src/lib/paths.ts index 01a5d92db1..71eff3d941 100644 --- a/packages/repo-tools/src/lib/paths.ts +++ b/packages/repo-tools/src/lib/paths.ts @@ -66,3 +66,39 @@ export async function findPackageDirs(selectedPaths: string[] = []) { } return packageDirs; } + +/** + * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root. + * + * If the file does not exist, or the "workspaces" field is not present, returns `undefined`. + * + * @returns The list of package names, or `undefined` if not found. + */ +export async function getWorkspacePackagePathPatterns(): Promise< + string[] | undefined +> { + const pkgJson = await fs + .readJson(paths.resolveTargetRoot('package.json')) + .catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + const workspaces = pkgJson?.workspaces?.packages; + return workspaces; +} + +/** + * Given a list of paths from the user or none and return the listing package directories from the + * workspace. + * @param cliPaths + * @returns + */ +export async function getMatchingWorkspacePaths(cliPaths: string[]) { + const isAllPackages = !cliPaths?.length; + const selectedPaths = isAllPackages + ? await getWorkspacePackagePathPatterns() + : cliPaths; + return await findPackageDirs(selectedPaths); +} diff --git a/yarn.lock b/yarn.lock index 3697631867..85cabd15e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3958,7 +3958,6 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/cli@workspace:packages/cli" dependencies: - "@apidevtools/swagger-parser": ^10.1.0 "@backstage/backend-common": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/config": "workspace:^" @@ -4050,7 +4049,6 @@ __metadata: jest-css-modules: ^2.1.0 jest-environment-jsdom: ^29.0.2 jest-runtime: ^29.0.2 - js-yaml: ^4.1.0 json-schema: ^0.4.0 lodash: ^4.17.21 mini-css-extract-plugin: ^2.4.2 @@ -4062,7 +4060,6 @@ __metadata: nodemon: ^2.0.2 npm-packlist: ^5.0.0 ora: ^5.3.0 - p-limit: ^3.0.2 postcss: ^8.1.0 process: ^0.11.10 react-dev-utils: ^12.0.0-next.60 @@ -9611,6 +9608,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/repo-tools@workspace:packages/repo-tools" dependencies: + "@apidevtools/swagger-parser": ^10.1.0 "@backstage/cli": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/errors": "workspace:^" @@ -9626,8 +9624,11 @@ __metadata: fs-extra: 10.1.0 glob: ^8.0.3 is-glob: ^4.0.3 + js-yaml: ^4.1.0 + lodash: ^4.17.21 minimatch: ^5.1.1 mock-fs: ^5.1.0 + p-limit: ^3.0.2 ts-node: ^10.0.0 peerDependencies: "@microsoft/api-extractor-model": "*"