Move CLI tooling to repo-tools.

Signed-off-by: Aramis Sennyey <sennyeya@amazon.com>

.changeset/fresh-lions-compete.md

@@ -1,5 +1,5 @@
 ---
-'@backstage/cli': minor
+'@backstage/repo-tools': minor
 ---
 
 Adding two new commands to support OpenAPI spec writing, `schema:openapi:generate` to generate the Typescript file that `@backstage/backend-openapi-utils` needs for typing and `schema:openapi:verify` to verify that this file exists and matches your `openapi.yaml` file.

.github/workflows/ci.yml

@@ -93,7 +93,7 @@ jobs:
         run: node scripts/verify-api-reference.js
 
       - name: verify openapi.yaml matches schema/openapi.ts files
-        run: yarn backstage-cli repo schema:openapi:verify
+        run: yarn backstage-repo-tools schema:openapi:verify
 
       - name: verify doc links
         run: node scripts/verify-links.js

packages/cli/cli-report.md

@@ -209,8 +209,6 @@ Commands:
   clean
   prepack
   postpack
-  schema:openapi:verify
-  schema:openapi:generate
   help [command]
 ```
 
@@ -267,24 +265,6 @@ Options:
   -h, --help
 ```
 
-### `backstage-cli package schema:openapi:generate`
-
-```
-Usage: backstage-cli package schema:openapi:generate [options]
-
-Options:
-  -h, --help
-```
-
-### `backstage-cli package schema:openapi:verify`
-
-```
-Usage: backstage-cli package schema:openapi:verify [options]
-
-Options:
-  -h, --help
-```
-
 ### `backstage-cli package start`
 
 ```
@@ -423,8 +403,6 @@ Commands:
   lint [options]
   clean
   list-deprecations [options]
-  schema:openapi:verify
-  schema:openapi:generate
   test [options]
   help [command]
 ```
@@ -471,24 +449,6 @@ Options:
   -h, --help
 ```
 
-### `backstage-cli repo schema:openapi:generate`
-
-```
-Usage: backstage-cli repo schema:openapi:generate [options]
-
-Options:
-  -h, --help
-```
-
-### `backstage-cli repo schema:openapi:verify`
-
-```
-Usage: backstage-cli repo schema:openapi:verify [options]
-
-Options:
-  -h, --help
-```
-
 ### `backstage-cli repo test`
 
 ```

packages/cli/package.json

@@ -30,7 +30,6 @@
     "backstage-cli": "bin/backstage-cli"
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
     "@backstage/cli-common": "workspace:^",
     "@backstage/config": "workspace:^",
     "@backstage/config-loader": "workspace:^",
@@ -97,7 +96,6 @@
     "jest-css-modules": "^2.1.0",
     "jest-environment-jsdom": "^29.0.2",
     "jest-runtime": "^29.0.2",
-    "js-yaml": "^4.1.0",
     "json-schema": "^0.4.0",
     "lodash": "^4.17.21",
     "mini-css-extract-plugin": "^2.4.2",
@@ -106,7 +104,6 @@
     "node-libs-browser": "^2.2.1",
     "npm-packlist": "^5.0.0",
     "ora": "^5.3.0",
-    "p-limit": "^3.0.2",
     "postcss": "^8.1.0",
     "process": "^0.11.10",
     "react-dev-utils": "^12.0.0-next.60",

packages/cli/src/commands/index.ts

@@ -80,20 +80,6 @@ export function registerRepoCommand(program: Command) {
       lazy(() => import('./repo/list-deprecations').then(m => m.command)),
     );
 
-  command
-    .command('schema:openapi:verify')
-    .description(
-      'Verify that all OpenAPI schemas are valid and have a matching `schemas/openapi.ts` file.',
-    )
-    .action(lazy(() => import('./openapi/verify').then(m => m.bulkCommand)));
-
-  command
-    .command('schema:openapi:generate')
-    .description(
-      'Generates a Typescript file from an OpenAPI yaml spec. For use with the `@backstage/backend-openapi-utils` ApiRouter type.',
-    )
-    .action(lazy(() => import('./openapi/generate').then(m => m.bulkCommand)));
-
   command
     .command('test')
     .allowUnknownOption(true) // Allows the command to run, but we still need to parse raw args
@@ -193,20 +179,6 @@ export function registerScriptCommand(program: Command) {
     .command('postpack')
     .description('Restores the changes made by the prepack command')
     .action(lazy(() => import('./pack').then(m => m.post)));
-
-  command
-    .command('schema:openapi:verify')
-    .description(
-      'Verifies that an OpenAPI spec is defined in both an `openapi.yaml` file and a `schema/openapi.ts` file.',
-    )
-    .action(lazy(() => import('./openapi/verify').then(m => m.command)));
-
-  command
-    .command('schema:openapi:generate')
-    .description(
-      'Generates a `schema/openapi.ts` file from an OpenAPI spec defined in `openapi.yaml`.',
-    )
-    .action(lazy(() => import('./openapi/generate').then(m => m.command)));
 }
 
 export function registerMigrateCommand(program: Command) {

packages/repo-tools/cli-report.md

@@ -14,6 +14,8 @@ Options:
 Commands:
   api-reports [options] [paths...]
   type-deps
+  schema:openapi:verify [paths...]
+  schema:openapi:generate [paths...]
   help [command]
 ```
 
@@ -33,6 +35,24 @@ Options:
   -h, --help
 ```
 
+### `backstage-repo-tools schema:openapi:generate`
+
+```
+Usage: backstage-repo-tools schema:openapi:generate [options] [paths...]
+
+Options:
+  -h, --help
+```
+
+### `backstage-repo-tools schema:openapi:verify`
+
+```
+Usage: backstage-repo-tools schema:openapi:verify [options] [paths...]
+
+Options:
+  -h, --help
+```
+
 ### `backstage-repo-tools type-deps`
 
 ```

packages/repo-tools/package.json

@@ -30,6 +30,7 @@
     "backstage-repo-tools": "bin/backstage-repo-tools"
   },
   "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
     "@backstage/cli-common": "workspace:^",
     "@backstage/errors": "workspace:^",
     "@manypkg/get-packages": "^1.1.3",
@@ -40,7 +41,10 @@
     "fs-extra": "10.1.0",
     "glob": "^8.0.3",
     "is-glob": "^4.0.3",
+    "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
     "minimatch": "^5.1.1",
+    "p-limit": "^3.0.2",
     "ts-node": "^10.0.0"
   },
   "devDependencies": {

packages/repo-tools/src/commands/api-reports/api-reports.ts

@@ -15,7 +15,6 @@
  */
 
 import { OptionValues } from 'commander';
-import fs from 'fs-extra';
 import {
   createTemporaryTsConfig,
   categorizePackageDirs,
@@ -23,7 +22,7 @@ import {
   runCliExtraction,
   buildDocs,
 } from './api-extractor';
-import { findPackageDirs, paths as cliPaths } from '../../lib/paths';
+import { getMatchingWorkspacePaths, paths as cliPaths } from '../../lib/paths';
 import { generateTypeDeclarations } from './generateTypeDeclarations';
 
 type Options = {
@@ -49,10 +48,7 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
   const omitMessages = parseArrayOption(opts.omitMessages);
 
   const isAllPackages = !paths?.length;
-  const selectedPaths = isAllPackages
-    ? await getWorkspacePackagePathPatterns()
-    : paths;
-  const selectedPackageDirs = await findPackageDirs(selectedPaths);
+  const selectedPackageDirs = await getMatchingWorkspacePaths(paths);
 
   if (isAllPackages && !isCiBuild && !isDocsBuild) {
     console.log('');
@@ -111,26 +107,6 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
   }
 };
 
-/**
- * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root.
- *
- * If the file does not exist, or the "workspaces" field is not present, returns `undefined`.
- *
- * @returns {Promise<string[] | undefined>} The list of package names, or `undefined` if not found.
- */
-async function getWorkspacePackagePathPatterns() {
-  const pkgJson = await fs
-    .readJson(cliPaths.resolveTargetRoot('package.json'))
-    .catch(error => {
-      if (error.code === 'ENOENT') {
-        return undefined;
-      }
-      throw error;
-    });
-  const workspaces = pkgJson?.workspaces?.packages;
-  return workspaces;
-}
-
 /**
  * Splits the input string on comma, and returns an array of the resulting substrings.
  * for `undefined` or an empty string, returns an empty array.

packages/repo-tools/src/commands/index.ts

@@ -52,6 +52,20 @@ export function registerCommands(program: Command) {
     .command('type-deps')
     .description('Find inconsistencies in types of all packages and plugins')
     .action(lazy(() => import('./type-deps/type-deps').then(m => m.default)));
+
+  program
+    .command('schema:openapi:verify [paths...]')
+    .description(
+      'Verify that all OpenAPI schemas are valid and have a matching `schemas/openapi.ts` file.',
+    )
+    .action(lazy(() => import('./openapi/verify').then(m => m.bulkCommand)));
+
+  program
+    .command('schema:openapi:generate [paths...]')
+    .description(
+      'Generates a Typescript file from an OpenAPI yaml spec. For use with the `@backstage/backend-openapi-utils` ApiRouter type.',
+    )
+    .action(lazy(() => import('./openapi/generate').then(m => m.bulkCommand)));
 }
 
 // Wraps an action function so that it always exits and handles errors

packages/repo-tools/src/commands/openapi/generate.ts

@@ -15,13 +15,10 @@
  */
 
 import fs from 'fs-extra';
-import { paths } from '../../lib/paths';
 import YAML from 'js-yaml';
 import chalk from 'chalk';
 import { resolve } from 'path';
-import { PackageGraph } from '../../lib/monorepo';
-import pLimit from 'p-limit';
-import { relative as relativePath } from 'path';
+import { runner } from './runner';
 
 async function generate(
   directoryPath: string,
@@ -44,36 +41,9 @@ async function generate(
   );
 }
 
-export async function command() {
-  try {
-    await generate(paths.resolveTarget('.'));
-    console.log(chalk.green('OpenAPI files successfully generated.'));
-  } catch (err) {
-    console.error(chalk.red(err.message));
-    process.exit(1);
-  }
-}
-
-export async function bulkCommand(): Promise<void> {
-  const packages = await PackageGraph.listTargetPackages();
-  const limit = pLimit(5);
-
-  const resultsList = await Promise.all(
-    packages.map(pkg =>
-      limit(async () => {
-        let resultText = '';
-        try {
-          await generate(pkg.dir, { skipMissingYamlFile: true });
-        } catch (err) {
-          resultText = err.message;
-        }
-
-        return {
-          relativeDir: relativePath(paths.targetRoot, pkg.dir),
-          resultText,
-        };
-      }),
-    ),
+export async function bulkCommand(paths: string[] = []): Promise<void> {
+  const resultsList = await runner(paths, (dir: string) =>
+    generate(dir, { skipMissingYamlFile: true }),
   );
 
   let failed = false;
@@ -93,5 +63,7 @@ export async function bulkCommand(): Promise<void> {
 
   if (failed) {
     process.exit(1);
+  } else {
+    console.log(chalk.green('Generated all files.'));
   }
 }

packages/repo-tools/src/commands/openapi/runner.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getMatchingWorkspacePaths } from '../../lib/paths';
+import pLimit from 'p-limit';
+import { relative as relativePath } from 'path';
+import { paths as cliPaths } from '../../lib/paths';
+
+export async function runner(
+  paths: string[],
+  command: (dir: string) => Promise<void>,
+) {
+  const packages = await getMatchingWorkspacePaths(paths);
+  const limit = pLimit(5);
+
+  const resultsList = await Promise.all(
+    packages.map(pkg =>
+      limit(async () => {
+        let resultText = '';
+        try {
+          console.log(`## Processing ${pkg}`);
+          await command(pkg);
+        } catch (err) {
+          resultText = err.message;
+        }
+
+        return {
+          relativeDir: relativePath(cliPaths.targetRoot, pkg),
+          resultText,
+        };
+      }),
+    ),
+  );
+
+  return resultsList;
+}

packages/repo-tools/src/commands/openapi/verify.ts

@@ -15,41 +15,16 @@
  */
 
 import fs from 'fs-extra';
-import { paths } from '../../lib/paths';
 import YAML from 'js-yaml';
-import { isEqual } from 'lodash';
-import { join, resolve } from 'path';
+import { isEqual, cloneDeep } from 'lodash';
+import { join } from 'path';
 import chalk from 'chalk';
-import { relative as relativePath } from 'path';
-import { PackageGraph } from '../../lib/monorepo';
-import { cloneDeep } from 'lodash';
+import { relative as relativePath, resolve as resolvePath } from 'path';
 import Parser from '@apidevtools/swagger-parser';
-import { detectRoleFromPackage } from '../../lib/role';
-import pLimit from 'p-limit';
-
-const SUPPORTED_ROLES = [
-  'backend',
-  'backend-plugin',
-  'backend-plugin-module',
-  'node-library',
-];
-
-async function verify(
-  directoryPath: string,
-  config: { checkRole: boolean } = { checkRole: false },
-) {
-  const { checkRole } = config ?? {};
-  if (checkRole) {
-    const role = detectRoleFromPackage(
-      await fs.readJson(resolve(directoryPath, 'package.json')),
-    );
-
-    if (!role || !SUPPORTED_ROLES.includes(role)) {
-      console.log(chalk.red(`Unsupported role ${role}`));
-      process.exit(1);
-    }
-  }
+import { runner } from './runner';
+import { paths as cliPaths } from '../../lib/paths';
 
+async function verify(directoryPath: string) {
   const openapiPath = join(directoryPath, 'openapi.yaml');
   if (!(await fs.pathExists(openapiPath))) {
     return;
@@ -62,51 +37,27 @@ async function verify(
   if (!(await fs.pathExists(schemaPath))) {
     throw new Error('No `schema/openapi.ts` file found.');
   }
-  const schema = await import(join(directoryPath, 'schema/openapi'));
+
+  const schema = await import(
+    resolvePath(join(directoryPath, 'schema/openapi'))
+  );
+
   if (!schema.default) {
     throw new Error('`schemas/openapi.ts` needs to have a default export.');
   }
   if (!isEqual(schema.default, yaml)) {
     throw new Error(
       `\`openapi.yaml\` and \`schema/openapi.ts\` do not match. Please run \`yarn --cwd ${relativePath(
-        paths.targetRoot,
+        cliPaths.targetRoot,
         directoryPath,
       )} schema:openapi:generate\` to regenerate \`schema/openapi.ts\`.`,
     );
   }
 }
 
-export async function command() {
-  try {
-    await verify(paths.resolveTarget('.'), { checkRole: true });
-    console.log(chalk.green('OpenAPI files are valid.'));
-  } catch (err) {
-    console.error(chalk.red(err.message));
-    process.exit(1);
-  }
-}
-
-export async function bulkCommand(): Promise<void> {
-  const packages = await PackageGraph.listTargetPackages();
-  const limit = pLimit(5);
-
-  const resultsList = await Promise.all(
-    packages.map(pkg =>
-      limit(async () => {
-        let resultText = '';
-        try {
-          await verify(pkg.dir);
-        } catch (err) {
-          resultText = err.message;
-        }
-
-        return {
-          relativeDir: relativePath(paths.targetRoot, pkg.dir),
-          resultText,
-        };
-      }),
-    ),
-  );
+export async function bulkCommand(paths: string[] = []): Promise<void> {
+  console.log(paths);
+  const resultsList = await runner(paths, dir => verify(dir));
 
   let failed = false;
   for (const { relativeDir, resultText } of resultsList) {
@@ -121,5 +72,7 @@ export async function bulkCommand(): Promise<void> {
 
   if (failed) {
     process.exit(1);
+  } else {
+    console.log(chalk.green('Verified all files.'));
   }
 }

packages/repo-tools/src/lib/paths.ts

@@ -66,3 +66,39 @@ export async function findPackageDirs(selectedPaths: string[] = []) {
   }
   return packageDirs;
 }
+
+/**
+ * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root.
+ *
+ * If the file does not exist, or the "workspaces" field is not present, returns `undefined`.
+ *
+ * @returns The list of package names, or `undefined` if not found.
+ */
+export async function getWorkspacePackagePathPatterns(): Promise<
+  string[] | undefined
+> {
+  const pkgJson = await fs
+    .readJson(paths.resolveTargetRoot('package.json'))
+    .catch(error => {
+      if (error.code === 'ENOENT') {
+        return undefined;
+      }
+      throw error;
+    });
+  const workspaces = pkgJson?.workspaces?.packages;
+  return workspaces;
+}
+
+/**
+ * Given a list of paths from the user or none and return the listing package directories from the
+ *  workspace.
+ * @param cliPaths
+ * @returns
+ */
+export async function getMatchingWorkspacePaths(cliPaths: string[]) {
+  const isAllPackages = !cliPaths?.length;
+  const selectedPaths = isAllPackages
+    ? await getWorkspacePackagePathPatterns()
+    : cliPaths;
+  return await findPackageDirs(selectedPaths);
+}

yarn.lock

@@ -3958,7 +3958,6 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@backstage/cli@workspace:packages/cli"
   dependencies:
-    "@apidevtools/swagger-parser": ^10.1.0
     "@backstage/backend-common": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/config": "workspace:^"
@@ -4050,7 +4049,6 @@ __metadata:
     jest-css-modules: ^2.1.0
     jest-environment-jsdom: ^29.0.2
     jest-runtime: ^29.0.2
-    js-yaml: ^4.1.0
     json-schema: ^0.4.0
     lodash: ^4.17.21
     mini-css-extract-plugin: ^2.4.2
@@ -4062,7 +4060,6 @@ __metadata:
     nodemon: ^2.0.2
     npm-packlist: ^5.0.0
     ora: ^5.3.0
-    p-limit: ^3.0.2
     postcss: ^8.1.0
     process: ^0.11.10
     react-dev-utils: ^12.0.0-next.60
@@ -9611,6 +9608,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@backstage/repo-tools@workspace:packages/repo-tools"
   dependencies:
+    "@apidevtools/swagger-parser": ^10.1.0
     "@backstage/cli": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/errors": "workspace:^"
@@ -9626,8 +9624,11 @@ __metadata:
     fs-extra: 10.1.0
     glob: ^8.0.3
     is-glob: ^4.0.3
+    js-yaml: ^4.1.0
+    lodash: ^4.17.21
     minimatch: ^5.1.1
     mock-fs: ^5.1.0
+    p-limit: ^3.0.2
     ts-node: ^10.0.0
   peerDependencies:
     "@microsoft/api-extractor-model": "*"