From 660e8cb2c12fe2b90ff0f4d2a781418cff56a9a4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 29 Dec 2024 11:35:38 +0100 Subject: [PATCH] repo-tools: refactor cli reports Signed-off-by: Patrik Oldsberg --- .../src/commands/api-reports/api-extractor.ts | 199 +----------------- .../commands/api-reports/api-reports.test.ts | 2 +- .../src/commands/api-reports/api-reports.ts | 2 +- .../cli-reports/generateCliReport.ts | 51 +++++ .../commands/api-reports/cli-reports/index.ts | 17 ++ .../cli-reports/runCliExtraction.ts | 171 +++++++++++++++ .../commands/api-reports/cli-reports/types.ts | 32 +++ 7 files changed, 274 insertions(+), 200 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/cli-reports/generateCliReport.ts create mode 100644 packages/repo-tools/src/commands/api-reports/cli-reports/index.ts create mode 100644 packages/repo-tools/src/commands/api-reports/cli-reports/runCliExtraction.ts create mode 100644 packages/repo-tools/src/commands/api-reports/cli-reports/types.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 0bac1cbeb1..54b12e623c 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -15,12 +15,7 @@ */ import { groupBy } from 'lodash'; -import { - basename, - join, - relative as relativePath, - resolve as resolvePath, -} from 'path'; +import { join, relative as relativePath, resolve as resolvePath } from 'path'; import fs from 'fs-extra'; import { CompilerState, @@ -1266,198 +1261,6 @@ export async function categorizePackageDirs(packageDirs: string[]) { return { tsPackageDirs, cliPackageDirs, sqlPackageDirs }; } -function parseHelpPage(helpPageContent: string) { - const [, usage] = helpPageContent.match(/^\s*Usage: (.*)$/im) ?? []; - const lines = helpPageContent.split(/\r?\n/); - - let options = new Array(); - let commands = new Array(); - let commandArguments = new Array(); - - while (lines.length > 0) { - while (lines.length > 0 && !lines[0].endsWith(':')) { - lines.shift(); - } - if (lines.length > 0) { - // Start of a new section, e.g. "Options:" - const sectionName = lines.shift(); - // Take lines until we hit the next section or the end - const sectionEndIndex = lines.findIndex( - line => line && !line.match(/^\s/), - ); - const sectionLines = lines.slice(0, sectionEndIndex); - lines.splice(0, sectionLines.length); - - // Trim away documentation - const sectionItems = sectionLines - .map(line => line.match(/^\s{1,8}(.*?)\s\s+/)?.[1]) - .filter(Boolean) as string[]; - - if (sectionName?.toLocaleLowerCase('en-US') === 'options:') { - options = sectionItems; - } else if (sectionName?.toLocaleLowerCase('en-US') === 'commands:') { - commands = sectionItems; - } else if (sectionName?.toLocaleLowerCase('en-US') === 'arguments:') { - commandArguments = sectionItems; - } else { - throw new Error(`Unknown CLI section: ${sectionName}`); - } - } - } - - return { - usage, - options, - commands, - commandArguments, - }; -} - -// Represents the help page os a CLI command -interface CliHelpPage { - // Path of commands to reach this page - path: string[]; - // Parsed content - usage: string | undefined; - options: string[]; - commands: string[]; - commandArguments: string[]; -} - -async function exploreCliHelpPages( - run: (...args: string[]) => Promise, -): Promise { - const helpPages = new Array(); - - async function exploreHelpPage(...path: string[]) { - const content = await run(...path, '--help'); - const parsed = parseHelpPage(content); - helpPages.push({ path, ...parsed }); - - await Promise.all( - parsed.commands.map(async fullCommand => { - const command = fullCommand.split(/[|\s]/)[0]; - if (command !== 'help') { - await exploreHelpPage(...path, command); - } - }), - ); - } - - await exploreHelpPage(); - - helpPages.sort((a, b) => a.path.join(' ').localeCompare(b.path.join(' '))); - - return helpPages; -} - -// The API model for a CLI entry point -interface CliModel { - name: string; - helpPages: CliHelpPage[]; -} - -function generateCliReport(name: string, models: CliModel[]): string { - const content = [ - `## CLI Report file for "${name}"`, - '', - '> Do not edit this file. It is a report generated by `yarn build:api-reports`', - '', - ]; - - for (const model of models) { - for (const helpPage of model.helpPages) { - content.push( - `### \`${[model.name, ...helpPage.path].join(' ')}\``, - '', - '```', - `Usage: ${helpPage.usage ?? ''}`, - ); - - if (helpPage.options.length > 0) { - content.push('', 'Options:', ...helpPage.options.map(l => ` ${l}`)); - } - - if (helpPage.commands.length > 0) { - content.push('', 'Commands:', ...helpPage.commands.map(l => ` ${l}`)); - } - content.push('```', ''); - } - } - - return content.join('\n'); -} - -interface CliExtractionOptions { - packageDirs: string[]; - isLocalBuild: boolean; -} - -export async function runCliExtraction({ - packageDirs, - isLocalBuild, -}: CliExtractionOptions) { - for (const packageDir of packageDirs) { - console.log(`## Processing ${packageDir}`); - const fullDir = cliPaths.resolveTargetRoot(packageDir); - const pkgJson = await fs.readJson(resolvePath(fullDir, 'package.json')); - - if (!pkgJson.bin) { - throw new Error(`CLI Package in ${packageDir} has no bin field`); - } - - const models = new Array(); - if (typeof pkgJson.bin === 'string') { - const run = createBinRunner(fullDir, pkgJson.bin); - const helpPages = await exploreCliHelpPages(run); - models.push({ name: basename(pkgJson.bin), helpPages }); - } else { - for (const [name, path] of Object.entries(pkgJson.bin)) { - const run = createBinRunner(fullDir, path); - const helpPages = await exploreCliHelpPages(run); - models.push({ name, helpPages }); - } - } - - const report = generateCliReport(pkgJson.name, models); - - const reportPath = resolvePath(fullDir, 'cli-report.md'); - const existingReport = await fs - .readFile(reportPath, 'utf8') - .catch(error => { - if (error.code === 'ENOENT') { - return undefined; - } - throw error; - }); - - if (existingReport !== report) { - if (isLocalBuild) { - console.warn(`CLI report changed for ${packageDir}`); - await fs.writeFile(reportPath, report); - } else { - logApiReportInstructions(); - - if (existingReport) { - console.log(''); - console.log( - `The conflicting file is ${relativePath( - cliPaths.targetRoot, - reportPath, - )}, expecting the following content:`, - ); - console.log(''); - - console.log(report); - - logApiReportInstructions(); - } - throw new Error(`CLI report changed for ${packageDir}, `); - } - } - } -} - interface KnipExtractionOptions { packageDirs: string[]; isLocalBuild: boolean; diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts index 02de908b8e..0a94b3449e 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -22,12 +22,12 @@ import { buildDocs, categorizePackageDirs, runApiExtraction, - runCliExtraction, } from './api-extractor'; import { buildApiReports } from './api-reports'; import { generateTypeDeclarations } from './generateTypeDeclarations'; import { PackageGraph } from '@backstage/cli-node'; +import { runCliExtraction } from './cli-reports'; jest.mock('./generateTypeDeclarations'); // create mocks for the dependencies of the `buildApiReports` function diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index bec9cfd6c3..93780bf86d 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -20,11 +20,11 @@ import { categorizePackageDirs, createTemporaryTsConfig, runApiExtraction, - runCliExtraction, } from './api-extractor'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; import { runSqlExtraction } from './sql-reports'; +import { runCliExtraction } from './cli-reports'; type Options = { ci?: boolean; diff --git a/packages/repo-tools/src/commands/api-reports/cli-reports/generateCliReport.ts b/packages/repo-tools/src/commands/api-reports/cli-reports/generateCliReport.ts new file mode 100644 index 0000000000..7df5058723 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/cli-reports/generateCliReport.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CliModel } from './types'; + +export function generateCliReport(options: { + packageName: string; + models: CliModel[]; +}): string { + const content = [ + `## CLI Report file for "${options.packageName}"`, + '', + '> Do not edit this file. It is a report generated by `yarn build:api-reports`', + '', + ]; + + for (const model of options.models) { + for (const helpPage of model.helpPages) { + content.push( + `### \`${[model.name, ...helpPage.path].join(' ')}\``, + '', + '```', + `Usage: ${helpPage.usage ?? ''}`, + ); + + if (helpPage.options.length > 0) { + content.push('', 'Options:', ...helpPage.options.map(l => ` ${l}`)); + } + + if (helpPage.commands.length > 0) { + content.push('', 'Commands:', ...helpPage.commands.map(l => ` ${l}`)); + } + content.push('```', ''); + } + } + + return content.join('\n'); +} diff --git a/packages/repo-tools/src/commands/api-reports/cli-reports/index.ts b/packages/repo-tools/src/commands/api-reports/cli-reports/index.ts new file mode 100644 index 0000000000..631cf41520 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/cli-reports/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runCliExtraction } from './runCliExtraction'; diff --git a/packages/repo-tools/src/commands/api-reports/cli-reports/runCliExtraction.ts b/packages/repo-tools/src/commands/api-reports/cli-reports/runCliExtraction.ts new file mode 100644 index 0000000000..cd630afec3 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/cli-reports/runCliExtraction.ts @@ -0,0 +1,171 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + basename, + resolve as resolvePath, + relative as relativePath, +} from 'path'; +import fs from 'fs-extra'; +import { createBinRunner } from '../../util'; +import { CliHelpPage, CliModel } from './types'; +import { paths as cliPaths } from '../../../lib/paths'; +import { generateCliReport } from './generateCliReport'; +import { logApiReportInstructions } from '../api-extractor'; + +function parseHelpPage(helpPageContent: string) { + const [, usage] = helpPageContent.match(/^\s*Usage: (.*)$/im) ?? []; + const lines = helpPageContent.split(/\r?\n/); + + let options = new Array(); + let commands = new Array(); + let commandArguments = new Array(); + + while (lines.length > 0) { + while (lines.length > 0 && !lines[0].endsWith(':')) { + lines.shift(); + } + if (lines.length > 0) { + // Start of a new section, e.g. "Options:" + const sectionName = lines.shift(); + // Take lines until we hit the next section or the end + const sectionEndIndex = lines.findIndex( + line => line && !line.match(/^\s/), + ); + const sectionLines = lines.slice(0, sectionEndIndex); + lines.splice(0, sectionLines.length); + + // Trim away documentation + const sectionItems = sectionLines + .map(line => line.match(/^\s{1,8}(.*?)\s\s+/)?.[1]) + .filter(Boolean) as string[]; + + if (sectionName?.toLocaleLowerCase('en-US') === 'options:') { + options = sectionItems; + } else if (sectionName?.toLocaleLowerCase('en-US') === 'commands:') { + commands = sectionItems; + } else if (sectionName?.toLocaleLowerCase('en-US') === 'arguments:') { + commandArguments = sectionItems; + } else { + throw new Error(`Unknown CLI section: ${sectionName}`); + } + } + } + + return { + usage, + options, + commands, + commandArguments, + }; +} + +async function exploreCliHelpPages( + run: (...args: string[]) => Promise, +): Promise { + const helpPages = new Array(); + + async function exploreHelpPage(...path: string[]) { + const content = await run(...path, '--help'); + const parsed = parseHelpPage(content); + helpPages.push({ path, ...parsed }); + + await Promise.all( + parsed.commands.map(async fullCommand => { + const command = fullCommand.split(/[|\s]/)[0]; + if (command !== 'help') { + await exploreHelpPage(...path, command); + } + }), + ); + } + + await exploreHelpPage(); + + helpPages.sort((a, b) => a.path.join(' ').localeCompare(b.path.join(' '))); + + return helpPages; +} + +interface CliExtractionOptions { + packageDirs: string[]; + isLocalBuild: boolean; +} + +export async function runCliExtraction({ + packageDirs, + isLocalBuild, +}: CliExtractionOptions) { + for (const packageDir of packageDirs) { + console.log(`## Processing ${packageDir}`); + const fullDir = cliPaths.resolveTargetRoot(packageDir); + const pkgJson = await fs.readJson(resolvePath(fullDir, 'package.json')); + + if (!pkgJson.bin) { + throw new Error(`CLI Package in ${packageDir} has no bin field`); + } + + const models = new Array(); + if (typeof pkgJson.bin === 'string') { + const run = createBinRunner(fullDir, pkgJson.bin); + const helpPages = await exploreCliHelpPages(run); + models.push({ name: basename(pkgJson.bin), helpPages }); + } else { + for (const [name, path] of Object.entries(pkgJson.bin)) { + const run = createBinRunner(fullDir, path); + const helpPages = await exploreCliHelpPages(run); + models.push({ name, helpPages }); + } + } + + const report = generateCliReport({ packageName: pkgJson.name, models }); + + const reportPath = resolvePath(fullDir, 'cli-report.md'); + const existingReport = await fs + .readFile(reportPath, 'utf8') + .catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + + if (existingReport !== report) { + if (isLocalBuild) { + console.warn(`CLI report changed for ${packageDir}`); + await fs.writeFile(reportPath, report); + } else { + logApiReportInstructions(); + + if (existingReport) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + cliPaths.targetRoot, + reportPath, + )}, expecting the following content:`, + ); + console.log(''); + + console.log(report); + + logApiReportInstructions(); + } + throw new Error(`CLI report changed for ${packageDir}, `); + } + } + } +} diff --git a/packages/repo-tools/src/commands/api-reports/cli-reports/types.ts b/packages/repo-tools/src/commands/api-reports/cli-reports/types.ts new file mode 100644 index 0000000000..27eb370564 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/cli-reports/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Represents the help page os a CLI command +export type CliHelpPage = { + // Path of commands to reach this page + path: string[]; + // Parsed content + usage: string | undefined; + options: string[]; + commands: string[]; + commandArguments: string[]; +}; + +// The API model for a CLI entry point +export type CliModel = { + name: string; + helpPages: CliHelpPage[]; +};