From c72656e7198f3d461727b2b86130c0904028288c Mon Sep 17 00:00:00 2001 From: djamaile Date: Mon, 14 Nov 2022 14:39:17 +0100 Subject: [PATCH 1/6] feat: introduce backstage repo tools package Signed-off-by: djamaile --- package.json | 3 + packages/repo-tools/.eslintrc.js | 6 + .../repo-tools/bin/backstage-repo-tools.js | 37 + packages/repo-tools/package.json | 51 + .../src/commands/api-reports/api-extractor.ts | 1345 +++++++++++++++++ .../src/commands/api-reports/api-reports.ts | 121 ++ packages/repo-tools/src/commands/index.ts | 55 + packages/repo-tools/src/index.ts | 53 + packages/repo-tools/src/lib/errors.ts | 48 + packages/repo-tools/src/tests/index.test.ts | 15 + yarn.lock | 15 + 11 files changed, 1749 insertions(+) create mode 100644 packages/repo-tools/.eslintrc.js create mode 100644 packages/repo-tools/bin/backstage-repo-tools.js create mode 100644 packages/repo-tools/package.json create mode 100644 packages/repo-tools/src/commands/api-reports/api-extractor.ts create mode 100644 packages/repo-tools/src/commands/api-reports/api-reports.ts create mode 100644 packages/repo-tools/src/commands/index.ts create mode 100644 packages/repo-tools/src/index.ts create mode 100644 packages/repo-tools/src/lib/errors.ts create mode 100644 packages/repo-tools/src/tests/index.test.ts diff --git a/package.json b/package.json index 2f20eeeee3..551a77a614 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "start-backend": "yarn workspace example-backend start", "build:backend": "yarn workspace backend build", "build:all": "backstage-cli repo build --all", + "build": "backstage-cli repo build --all", + "build:api-reports-tooling": "backstage-repo-tools api-reports", "build:api-reports": "yarn build:api-reports:only --tsc", "build:api-reports:only": "ts-node -T -P scripts/tsconfig.json scripts/api-extractor.ts", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs", @@ -59,6 +61,7 @@ "@backstage/cli": "workspace:*", "@backstage/codemods": "workspace:*", "@backstage/create-app": "workspace:*", + "@backstage/repo-tools": "workspace:*", "@changesets/cli": "^2.14.0", "@octokit/rest": "^19.0.3", "@spotify/prettier-config": "^14.0.0", diff --git a/packages/repo-tools/.eslintrc.js b/packages/repo-tools/.eslintrc.js new file mode 100644 index 0000000000..e9ea2f5bfb --- /dev/null +++ b/packages/repo-tools/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { + ignorePatterns: ['templates/**'], + rules: { + 'no-console': 0, + }, +}); diff --git a/packages/repo-tools/bin/backstage-repo-tools.js b/packages/repo-tools/bin/backstage-repo-tools.js new file mode 100644 index 0000000000..377e4a32ca --- /dev/null +++ b/packages/repo-tools/bin/backstage-repo-tools.js @@ -0,0 +1,37 @@ +#!/usr/bin/env node +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); + +// Figure out whether we're running inside the backstage repo or as an installed dependency +/* eslint-disable-next-line no-restricted-syntax */ +const isLocal = require('fs').existsSync(path.resolve(__dirname, '../src')); + +if (!isLocal || process.env.BACKSTAGE_E2E_CLI_TEST) { + require('..'); +} else { + require('ts-node').register({ + transpileOnly: true, + /* eslint-disable-next-line no-restricted-syntax */ + project: path.resolve(__dirname, '../../../tsconfig.json'), + compilerOptions: { + module: 'CommonJS', + }, + }); + + require('../src'); +} diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json new file mode 100644 index 0000000000..480e57b25a --- /dev/null +++ b/packages/repo-tools/package.json @@ -0,0 +1,51 @@ +{ + "name": "@backstage/repo-tools", + "description": "", + "version": "0.0.0", + "publishConfig": { + "access": "public" + }, + "backstage": { + "role": "cli" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/repo-tools" + }, + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", + "main": "dist/index.cjs.js", + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "start": "nodemon --" + }, + "bin": { + "backstage-repo-tools": "bin/backstage-repo-tools" + }, + "dependencies": { + "@backstage/errors": "workspace:^", + "chalk": "^4.0.0", + "commander": "^9.1.0", + "fs-extra": "10.1.0", + "ts-node": "^10.0.0" + }, + "files": [ + "asset-types", + "templates", + "config", + "bin", + "dist/**/*.js" + ], + "nodemonConfig": { + "watch": "./src", + "exec": "bin/backstage-repo-tools", + "ext": "ts" + } +} diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts new file mode 100644 index 0000000000..85a9310ab4 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -0,0 +1,1345 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable import/no-extraneous-dependencies */ +/* eslint-disable no-restricted-imports */ + +import { + resolve as resolvePath, + relative as relativePath, + basename, + dirname, + join, +} from 'path'; +import { spawnSync, execFile } from 'child_process'; +import prettier from 'prettier'; +import fs from 'fs-extra'; +import { + Extractor, + ExtractorConfig, + CompilerState, + ExtractorLogLevel, + ExtractorMessage, +} from '@microsoft/api-extractor'; +import { Program } from 'typescript'; +import { + DocNode, + IDocNodeContainerParameters, + TSDocTagSyntaxKind, +} from '@microsoft/tsdoc'; +import { TSDocConfigFile } from '@microsoft/tsdoc-config'; +import { ApiPackage, ApiModel, ApiItem } from '@microsoft/api-extractor-model'; +import { + IMarkdownDocumenterOptions, + MarkdownDocumenter, +} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter'; +import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable'; +import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow'; +import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading'; +import { CustomMarkdownEmitter } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; +import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; +import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; + +const tmpDir = resolvePath(process.cwd(), './node_modules/.cache/api-extractor'); + +/** + * All of this monkey patching below is because MUI has these bare package.json file as a method + * for making TypeScript accept imports like `@material-ui/core/Button`, and improve tree-shaking + * by declaring them side effect free. + * + * The package.json lookup logic in api-extractor really doesn't like that though, as it enforces + * that the 'name' field exists in all package.json files that it discovers. This below is just + * making sure that we ignore those file package.json files instead of crashing. + */ +const { + PackageJsonLookup, +} = require('@rushstack/node-core-library/lib/PackageJsonLookup'); + +const old = PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor; +PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = + function tryGetPackageJsonFilePathForPatch(path: string) { + if ( + path.includes('@material-ui') && + !dirname(path).endsWith('@material-ui') + ) { + return undefined; + } + return old.call(this, path); + }; + +/** + * Another monkey patch where we apply prettier to the API reports. This has to be patched into + * the middle of the process as API Extractor does a comparison of the contents of the old + * and new files during generation. This inserts the formatting just before that comparison. + */ +const { + ApiReportGenerator, +} = require('@microsoft/api-extractor/lib/generators/ApiReportGenerator'); + +function patchFileMessageFetcher( + router: any, + transform: (messages: ExtractorMessage[], ast?: AstDeclaration) => void, +) { + const { + fetchAssociatedMessagesForReviewFile, + fetchUnassociatedMessagesForReviewFile, + } = router; + + router.fetchAssociatedMessagesForReviewFile = + function patchedFetchAssociatedMessagesForReviewFile(ast) { + const messages = fetchAssociatedMessagesForReviewFile.call(this, ast); + return transform(messages, ast); + }; + router.fetchUnassociatedMessagesForReviewFile = + function patchedFetchUnassociatedMessagesForReviewFile() { + const messages = fetchUnassociatedMessagesForReviewFile.call(this); + return transform(messages); + }; +} + +const originalGenerateReviewFileContent = + ApiReportGenerator.generateReviewFileContent; +ApiReportGenerator.generateReviewFileContent = + function decoratedGenerateReviewFileContent(collector, ...moreArgs) { + const program = collector.program as Program; + + // The purpose of this override is to allow the @ignore tag to be used to ignore warnings + // of the form "Warning: (ae-forgotten-export) The symbol "FooBar" needs to be exported by the entry point index.d.ts" + patchFileMessageFetcher( + collector.messageRouter, + (messages: ExtractorMessage[]) => { + return messages.filter(message => { + if (message.messageId !== 'ae-forgotten-export') { + return true; + } + + // Symbol name has to be extracted from the message :( + // There's frequently no AST for these exports because type literals + // aren't traversed by the generator. + const symbolMatch = message.text.match(/The symbol "([^"]+)"/); + if (!symbolMatch) { + throw new Error( + `Failed to extract symbol name from message "${message.text}"`, + ); + } + const [, symbolName] = symbolMatch; + + const sourceFile = program.getSourceFile(message.sourceFilePath); + if (!sourceFile) { + throw new Error( + `Failed to find source file in program at path "${message.sourceFilePath}"`, + ); + } + + // The local name of the symbol within the file, rather than the exported name + const localName = (sourceFile as any).identifiers?.get(symbolName); + if (!localName) { + throw new Error( + `Unable to find local name of "${symbolName}" in ${sourceFile.fileName}`, + ); + } + + // The local AST node of the export that we're missing + const local = (sourceFile as any).locals?.get(localName); + if (!local) { + return true; + } + + // Use the type checker to look up the actual declaration(s) rather than the one in the local file + const type = program.getTypeChecker().getDeclaredTypeOfSymbol(local); + if (!type) { + throw new Error( + `Unable to find type declaration of "${symbolName}" in ${sourceFile.fileName}`, + ); + } + const declarations = type.aliasSymbol?.declarations; + if (!declarations || declarations.length === 0) { + return true; + } + + // If any of the TSDoc comments contain a @ignore tag, we ignore this message + const isIgnored = declarations.some(declaration => { + const tags = [(declaration as any).jsDoc] + .flat() + .filter(Boolean) + .flatMap((tagNode: any) => tagNode.tags); + + return tags.some(tag => tag?.tagName.text === 'ignore'); + }); + + return !isIgnored; + }); + }, + ); + + const content = originalGenerateReviewFileContent.call( + this, + collector, + ...moreArgs, + ); + return prettier.format(content, { + ...require('@spotify/prettier-config'), + parser: 'markdown', + }); + }; + +const PACKAGE_ROOTS = ['packages', 'plugins']; + +const ALLOW_WARNINGS = [ + 'packages/core-components', + 'plugins/catalog', + 'plugins/catalog-import', + 'plugins/git-release-manager', + 'plugins/jenkins', + 'plugins/kubernetes', +]; + +async function resolvePackagePath( + packagePath: string, +): Promise { + const projectRoot = resolvePath(process.cwd()); + const fullPackageDir = resolvePath(projectRoot, packagePath); + + const stat = await fs.stat(fullPackageDir); + if (!stat.isDirectory()) { + return undefined; + } + + try { + const packageJsonPath = join(fullPackageDir, 'package.json'); + await fs.access(packageJsonPath); + } catch (_) { + return undefined; + } + + return relativePath(projectRoot, fullPackageDir); +} + +export async function findSpecificPackageDirs(unresolvedPackageDirs: string[]) { + const packageDirs = new Array(); + + for (const unresolvedPackageDir of unresolvedPackageDirs) { + const packageDir = await resolvePackagePath(unresolvedPackageDir); + if (!packageDir) { + throw new Error(`'${unresolvedPackageDir}' is not a valid package path`); + } + packageDirs.push(packageDir); + } + + if (packageDirs.length === 0) { + return undefined; + } + + return packageDirs; +} + +export async function findPackageDirs() { + const packageDirs = new Array(); + const projectRoot = resolvePath(process.cwd()); + + for (const packageRoot of PACKAGE_ROOTS) { + const dirs = await fs.readdir(resolvePath(projectRoot, packageRoot)); + for (const dir of dirs) { + const packageDir = await resolvePackagePath(join(packageRoot, dir)); + if (!packageDir) { + continue; + } + + packageDirs.push(packageDir); + } + } + + return packageDirs; +} + +export async function createTemporaryTsConfig(includedPackageDirs: string[]) { + const path = resolvePath(process.cwd(), 'tsconfig.tmp.json'); + + process.once('exit', () => { + fs.removeSync(path); + }); + + await fs.writeJson(path, { + extends: './tsconfig.json', + include: [ + // These two contain global definitions that are needed for stable API report generation + 'packages/cli/asset-types/asset-types.d.ts', + ...includedPackageDirs.map(dir => join(dir, 'src')), + ], + }); + + return path; +} + +export async function countApiReportWarnings(projectFolder: string) { + const path = resolvePath(projectFolder, 'api-report.md'); + try { + const content = await fs.readFile(path, 'utf8'); + const lines = content.split('\n'); + + const lineWarnings = lines.filter(line => + line.includes('// Warning:'), + ).length; + + const trailerStart = lines.findIndex( + line => line === '// Warnings were encountered during analysis:', + ); + const trailerWarnings = + trailerStart === -1 + ? 0 + : lines.length - + trailerStart - + 4; /* 4 lines at the trailer and after are not warnings */ + + return lineWarnings + trailerWarnings; + } catch (error) { + if (error.code === 'ENOENT') { + return 0; + } + throw error; + } +} + +export async function getTsDocConfig() { + const tsdocConfigFile = await TSDocConfigFile.loadFile( + require.resolve('@microsoft/api-extractor/extends/tsdoc-base.json'), + ); + tsdocConfigFile.addTagDefinition({ + tagName: '@ignore', + syntaxKind: TSDocTagSyntaxKind.ModifierTag, + }); + tsdocConfigFile.setSupportForTag('@ignore', true); + return tsdocConfigFile; +} + +function logApiReportInstructions() { + console.log(''); + console.log( + '*************************************************************************************', + ); + console.log( + '* You have uncommitted changes to the public API of a package. *', + ); + console.log( + '* To solve this, run `yarn build:api-reports` and commit all api-report.md changes. *', + ); + console.log( + '*************************************************************************************', + ); + console.log(''); +} + +interface ApiExtractionOptions { + packageDirs: string[]; + outputDir: string; + isLocalBuild: boolean; + tsconfigFilePath: string; +} + +export async function runApiExtraction({ + packageDirs, + outputDir, + isLocalBuild, + tsconfigFilePath, +}: ApiExtractionOptions) { + await fs.remove(outputDir); + + const entryPoints = packageDirs.map(packageDir => { + return resolvePath(process.cwd(), `./dist-types/${packageDir}/src/index.d.ts`); + }); + + let compilerState: CompilerState | undefined = undefined; + + const warnings = new Array(); + + for (const packageDir of packageDirs) { + console.log(`## Processing ${packageDir}`); + const projectFolder = resolvePath(process.cwd(), packageDir); + const packageFolder = resolvePath(process.cwd(), './dist-types', packageDir); + + const warningCountBefore = await countApiReportWarnings(projectFolder); + + const extractorConfig = ExtractorConfig.prepare({ + configObject: { + mainEntryPointFilePath: resolvePath(packageFolder, 'src/index.d.ts'), + bundledPackages: [], + + compiler: { + tsconfigFilePath, + }, + + apiReport: { + enabled: true, + reportFileName: 'api-report.md', + reportFolder: projectFolder, + reportTempFolder: resolvePath(outputDir, ''), + }, + + docModel: { + enabled: true, + apiJsonFilePath: resolvePath( + outputDir, + '.api.json', + ), + }, + + dtsRollup: { + enabled: false, + }, + + tsdocMetadata: { + enabled: false, + }, + + messages: { + // Silence compiler warnings, as these will prevent the CI build to work + compilerMessageReporting: { + default: { + logLevel: 'none' as ExtractorLogLevel.None, + // These contain absolute file paths, so can't be included in the report + // addToApiReportFile: true, + }, + }, + extractorMessageReporting: { + default: { + logLevel: 'warning' as ExtractorLogLevel.Warning, + addToApiReportFile: true, + }, + }, + tsdocMessageReporting: { + default: { + logLevel: 'warning' as ExtractorLogLevel.Warning, + addToApiReportFile: true, + }, + }, + }, + + newlineKind: 'lf', + + projectFolder, + }, + configObjectFullPath: projectFolder, + packageJsonFullPath: resolvePath(projectFolder, 'package.json'), + tsdocConfigFile: await getTsDocConfig(), + }); + + // The `packageFolder` needs to point to the location within `dist-types` in order for relative + // paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`, + // which needs to point to the actual file, so we override `packageFolder` afterwards. + ( + extractorConfig as { + packageFolder: string; + } + ).packageFolder = packageFolder; + + if (!compilerState) { + compilerState = CompilerState.create(extractorConfig, { + additionalEntryPoints: entryPoints, + }); + } + + // Message verbosity can't be configured, so just skip the check instead + (Extractor as any)._checkCompilerCompatibility = () => {}; + + let shouldLogInstructions = false; + let conflictingFile: undefined | string = undefined; + + // Invoke API Extractor + const extractorResult = Extractor.invoke(extractorConfig, { + localBuild: isLocalBuild, + showVerboseMessages: false, + showDiagnostics: false, + messageCallback(message) { + if ( + message.text.includes( + 'You have changed the public API signature for this project.', + ) + ) { + shouldLogInstructions = true; + const match = message.text.match( + /Please copy the file "(.*)" to "api-report\.md"/, + ); + if (match) { + conflictingFile = match[1]; + } + } + }, + compilerState, + }); + + if (!extractorResult.succeeded) { + if (shouldLogInstructions) { + logApiReportInstructions(); + + if (conflictingFile) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + tmpDir, + conflictingFile, + )}, with the following content:`, + ); + console.log(''); + + const content = await fs.readFile(conflictingFile, 'utf8'); + console.log(content); + + logApiReportInstructions(); + } + } + + throw new Error( + `API Extractor completed with ${extractorResult.errorCount} errors` + + ` and ${extractorResult.warningCount} warnings`, + ); + } + + const warningCountAfter = await countApiReportWarnings(projectFolder); + if (warningCountAfter > 0 && !ALLOW_WARNINGS.includes(packageDir)) { + throw new Error( + `The API Report for ${packageDir} is not allowed to have warnings`, + ); + } + if (warningCountAfter === 0 && ALLOW_WARNINGS.includes(packageDir)) { + console.log( + `No need to allow warnings for ${packageDir}, it does not have any`, + ); + } + if (warningCountAfter > warningCountBefore) { + warnings.push( + `The API Report for ${packageDir} introduces new warnings. ` + + 'Please fix these warnings in order to keep the API Reports tidy.', + ); + } + } + + if (warnings.length > 0) { + console.warn(); + for (const warning of warnings) { + console.warn(warning); + } + console.warn(); + } +} + +/* +WARNING: Bring a blanket if you're gonna read the code below + +There's some weird shit going on here, and it's because we cba +forking rushstack to modify the api-documenter markdown generation, +which otherwise is the recommended way to do customizations. +*/ + +type ExcerptToken = { + kind: string; + text: string; + canonicalReference?: string; +}; + +class ExcerptTokenMatcher { + readonly #tokens: ExcerptToken[]; + + constructor(tokens: ExcerptToken[]) { + this.#tokens = tokens.slice(); + } + + nextContent() { + const token = this.#tokens.shift(); + if (token?.kind === 'Content') { + return token.text; + } + return undefined; + } + + matchContent(expectedText: string) { + const text = this.nextContent(); + return text !== expectedText; + } + + getTokensUntilArrow() { + const tokens = []; + for (;;) { + const token = this.#tokens.shift(); + if (token === undefined) { + return undefined; + } + if (token.kind === 'Content' && token.text === ') => ') { + return tokens; + } + tokens.push(token); + } + } + + getComponentReturnTokens() { + const first = this.#tokens.shift(); + if (!first) { + return undefined; + } + const second = this.#tokens.shift(); + + if (this.#tokens.length !== 0) { + return undefined; + } + if (first.kind !== 'Reference' || first.text !== 'JSX.Element') { + return undefined; + } + if (!second) { + return [first]; + } else if (second.kind === 'Content' && second.text === ' | null') { + return [first, second]; + } + return undefined; + } +} + +class ApiModelTransforms { + static deserializeWithTransforms( + serialized: any, + transforms: Array<(member: any) => any>, + ): ApiPackage { + if (serialized.kind !== 'Package') { + throw new Error( + `Unexpected root kind in serialized ApiPackage, ${serialized.kind}`, + ); + } + if (serialized.members.length !== 1) { + throw new Error( + `Unexpected members in serialized ApiPackage, [${serialized.members + .map(m => m.kind) + .join(' ')}]`, + ); + } + const [entryPoint] = serialized.members; + if (entryPoint.kind !== 'EntryPoint') { + throw new Error( + `Unexpected kind in serialized ApiPackage member, ${entryPoint.kind}`, + ); + } + + const transformed = { + ...serialized, + members: [ + { + ...entryPoint, + members: entryPoint.members.map(member => + transforms.reduce((m, t) => t(m), member), + ), + }, + ], + }; + + return ApiPackage.deserialize( + transformed, + transformed.metadata, + ) as ApiPackage; + } + + static transformArrowComponents = (member: any) => { + if (member.kind !== 'Variable') { + return member; + } + + const { name, excerptTokens } = member; + + // First letter in name must be uppercase + const [firstChar] = name; + if (firstChar.toLocaleUpperCase('en-US') !== firstChar) { + return member; + } + + // First content must match expected declaration format + const tokens = new ExcerptTokenMatcher(excerptTokens); + if (tokens.nextContent() !== `${name}: `) { + return member; + } + + // Next needs to be an arrow with `props` parameters or no parameters + // followed by a return type of `JSX.Element | null` or just `JSX.Element` + const declStart = tokens.nextContent(); + if (declStart === '(props: ' || declStart === '(_props: ') { + const props = tokens.getTokensUntilArrow(); + const ret = tokens.getComponentReturnTokens(); + if (props && ret) { + return this.makeComponentMember(member, ret, props); + } + } else if (declStart === '() => ') { + const ret = tokens.getComponentReturnTokens(); + if (ret) { + return this.makeComponentMember(member, ret); + } + } + return member; + }; + + static makeComponentMember( + member: any, + ret: ExcerptToken[], + props?: ExcerptToken[], + ) { + const declTokens = props + ? [ + { + kind: 'Content', + text: `export declare function ${member.name}(props: `, + }, + ...props, + { + kind: 'Content', + text: '): ', + }, + ] + : [ + { + kind: 'Content', + text: `export declare function ${member.name}(): `, + }, + ]; + + return { + kind: 'Function', + name: member.name, + releaseTag: member.releaseTag, + docComment: member.docComment ?? '', + canonicalReference: member.canonicalReference, + excerptTokens: [...declTokens, ...ret], + returnTypeTokenRange: { + startIndex: declTokens.length, + endIndex: declTokens.length + ret.length, + }, + parameters: props + ? [ + { + parameterName: 'props', + parameterTypeTokenRange: { + startIndex: 1, + endIndex: 1 + props.length, + }, + }, + ] + : [], + overloadIndex: 1, + }; + } + + static transformTrimDeclare = (member: any) => { + const { excerptTokens } = member; + const firstContent = new ExcerptTokenMatcher(excerptTokens).nextContent(); + if (firstContent && firstContent.startsWith('export declare ')) { + return { + ...member, + excerptTokens: [ + { + kind: 'Content', + text: firstContent.slice('export declare '.length), + }, + ...excerptTokens.slice(1), + ], + }; + } + return member; + }; +} + +export async function buildDocs({ + inputDir, + outputDir, +}: { + inputDir: string; + outputDir: string; +}) { + // We start by constructing our own model from the files so that + // we get a change to modify them, as the model is otherwise read-only. + const parseFile = async (filename: string): Promise => { + console.log(`Reading ${filename}`); + return fs.readJson(resolvePath(inputDir, filename)); + }; + + const filenames = await fs.readdir(inputDir); + const serializedPackages = await Promise.all( + filenames + .filter(filename => filename.match(/\.api\.json$/i)) + .map(parseFile), + ); + + const newModel = new ApiModel(); + for (const serialized of serializedPackages) { + newModel.addMember( + ApiModelTransforms.deserializeWithTransforms(serialized, [ + ApiModelTransforms.transformArrowComponents, + ApiModelTransforms.transformTrimDeclare, + ]), + ); + } + + // The doc AST need to be extended with custom nodes if we want to + // add any extra content. + // This one is for the YAML front matter that we need for the microsite. + class DocFrontMatter extends DocNode { + static kind = 'DocFrontMatter'; + + public readonly values: { [name: string]: unknown }; + + public constructor( + parameters: IDocNodeContainerParameters & { + values: { [name: string]: unknown }; + }, + ) { + super(parameters); + this.values = parameters.values; + } + + /** @override */ + public get kind(): string { + return DocFrontMatter.kind; + } + } + + // This is where we actually write the markdown and where we can hook + // in the rendering of our own nodes. + class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter { + // Until https://github.com/microsoft/rushstack/issues/2914 gets fixed or we change markdown renderer we need a fix + // to render pipe | character correctly. + protected getEscapedText(text: string): string { + return text + .replace(/\\/g, '\\\\') // first replace the escape character + .replace(/[*#[\]_`~]/g, x => `\\${x}`) // then escape any special characters + .replace(/---/g, '\\-\\-\\-') // hyphens only if it's 3 or more + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/\|/g, '|'); + } + /** @override */ + protected writeNode( + docNode: DocNode, + context: IMarkdownEmitterContext, + docNodeSiblings: boolean, + ): void { + switch (docNode.kind) { + case DocFrontMatter.kind: { + const node = docNode as DocFrontMatter; + context.writer.writeLine('---'); + for (const [name, value] of Object.entries(node.values)) { + if (value) { + context.writer.writeLine(`${name}: ${value}`); + } + } + context.writer.writeLine('---'); + context.writer.writeLine(); + break; + } + default: + super.writeNode(docNode, context, docNodeSiblings); + } + } + + /** @override */ + emit(stringBuilder, docNode, options) { + // Hack to get rid of the leading comment of each file, since + // we want the front matter to come first + stringBuilder._chunks.length = 0; + return super.emit(stringBuilder, docNode, options); + } + } + + class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) { + constructor(options: IMarkdownDocumenterOptions) { + super(options); + + // It's a strict model, we gotta register the allowed usage of our new node + this._tsdocConfiguration.docNodeManager.registerDocNodes( + '@backstage/docs', + [{ docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter }], + ); + this._tsdocConfiguration.docNodeManager.registerAllowableChildren( + 'Paragraph', + [DocFrontMatter.kind], + ); + + this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel); + } + + private _getFilenameForApiItem(apiItem: ApiItem): string { + const filename: string = super._getFilenameForApiItem(apiItem); + + if (filename.includes('.html.')) { + return filename.replace(/\.html\./g, '._html.'); + } + + return filename; + } + + // We don't really get many chances to modify the generated AST + // so we hook in wherever we can. In this case we add the front matter + // just before writing the breadcrumbs at the top. + /** @override */ + _writeBreadcrumb(output, apiItem) { + let title; + let description; + + const name = apiItem.getScopedNameWithinPackage(); + if (name) { + title = name; + description = `API reference for ${apiItem.getScopedNameWithinPackage()}`; + } else if (apiItem.kind === 'Model') { + title = 'Package Index'; + description = 'Index of all Backstage Packages'; + } else { + title = apiItem.name; + description = `API Reference for ${apiItem.name}`; + } + + // Add our front matter + output.appendNodeInParagraph( + new DocFrontMatter({ + configuration: this._tsdocConfiguration, + values: { + id: this._getFilenameForApiItem(apiItem).slice(0, -3), + title, + description, + }, + }), + ); + + // Now write the actual breadcrumbs + super._writeBreadcrumb(output, apiItem); + + // We wanna ignore the header that always gets written after the breadcrumb + // This otherwise becomes more or less a duplicate of the title in the front matter + const oldAppendNode = output.appendNode; + output.appendNode = () => { + output.appendNode = oldAppendNode; + }; + } + + _writeModelTable(output, apiModel): void { + const configuration = this._tsdocConfiguration; + + const packagesTable = new DocTable({ + configuration, + headerTitles: ['Package', 'Description'], + }); + + const pluginsTable = new DocTable({ + configuration, + headerTitles: ['Package', 'Description'], + }); + + for (const apiMember of apiModel.members) { + const row = new DocTableRow({ configuration }, [ + this._createTitleCell(apiMember), + this._createDescriptionCell(apiMember), + ]); + + if (apiMember.kind === 'Package') { + this._writeApiItemPage(apiMember); + + if (apiMember.name.startsWith('@backstage/plugin-')) { + pluginsTable.addRow(row); + } else { + packagesTable.addRow(row); + } + } + } + + if (packagesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Packages', + }), + ); + output.appendNode(packagesTable); + } + + if (pluginsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Plugins', + }), + ); + output.appendNode(pluginsTable); + } + } + } + + // This is root of the documentation generation, but it's not directly + // responsible for generating markdown, it just constructs an AST that + // is the consumed by an emitter to actually write the files. + const documenter = new CustomMarkdownDocumenter({ + apiModel: newModel, + documenterConfig: { + outputTarget: 'markdown', + newlineKind: '\n', + // De ba dålig kod + configFilePath: '', + configFile: {}, + } as any, + outputFolder: outputDir, + }); + + // Clean up existing stuff and write ALL the docs! + await fs.remove(outputDir); + await fs.ensureDir(outputDir); + documenter.generateFiles(); +} + +export async function categorizePackageDirs(projectRoot, packageDirs) { + const dirs = packageDirs.slice(); + const tsPackageDirs = new Array(); + const cliPackageDirs = new Array(); + + await Promise.all( + Array(10) + .fill(0) + .map(async () => { + for (;;) { + const dir = dirs.pop(); + if (!dir) { + return; + } + + const pkgJson = await fs + .readJson(resolvePath(projectRoot, dir, 'package.json')) + .catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + const role = pkgJson?.backstage?.role; + if (!role) { + throw new Error(`No backstage.role in ${dir}/package.json`); + } + if (role === 'cli') { + cliPackageDirs.push(dir); + } else if (role !== 'frontend' && role !== 'backend') { + tsPackageDirs.push(dir); + } + } + }), + ); + + return { tsPackageDirs, cliPackageDirs }; +} + +function createBinRunner(cwd: string, path: string) { + return async (...command: string[]) => + new Promise((resolve, reject) => { + execFile( + 'node', + [path, ...command], + { + cwd, + shell: true, + timeout: 60000, + maxBuffer: 1024 * 1024, + }, + (err, stdout, stderr) => { + if (err) { + reject(new Error(`${err.message}\n${stderr}`)); + } else if (stderr) { + reject(new Error(`Command printed error output: ${stderr}`)); + } else { + resolve(stdout); + } + }, + ); + }); +} + +function parseHelpPage(helpPageContent: string) { + const [, usage] = helpPageContent.match(/^\s*Usage: (.*)$/im) ?? []; + const lines = helpPageContent.split(/\r?\n/); + + let options = new Array(); + let commands = new Array(); + + while (lines.length > 0) { + while (lines.length > 0 && !lines[0].endsWith(':')) { + lines.shift(); + } + if (lines.length > 0) { + // Start of a new section, e.g. "Options:" + const sectionName = lines.shift(); + // Take lines until we hit the next section or the end + const sectionEndIndex = lines.findIndex( + line => line && !line.match(/^\s/), + ); + const sectionLines = lines.slice(0, sectionEndIndex); + lines.splice(0, sectionLines.length); + + // Trim away documentation + const sectionItems = sectionLines + .map(line => line.match(/^\s{1,8}(.*?)\s\s+/)?.[1]) + .filter(Boolean); + + if (sectionName.toLocaleLowerCase('en-US') === 'options:') { + options = sectionItems; + } else if (sectionName.toLocaleLowerCase('en-US') === 'commands:') { + commands = sectionItems; + } else { + throw new Error(`Unknown CLI section: ${sectionName}`); + } + } + } + + return { + usage, + options, + commands, + }; +} + +// Represents the help page os a CLI command +interface CliHelpPage { + // Path of commands to reach this page + path: string[]; + // Parsed content + usage: string | undefined; + options: string[]; + commands: string[]; +} + +async function exploreCliHelpPages( + run: (...args: string[]) => Promise, +): Promise { + const helpPages = new Array(); + + async function exploreHelpPage(...path: string[]) { + const content = await run(...path, '--help'); + const parsed = parseHelpPage(content); + helpPages.push({ path, ...parsed }); + + await Promise.all( + parsed.commands.map(async fullCommand => { + const command = fullCommand.split(/[|\s]/)[0]; + if (command !== 'help') { + await exploreHelpPage(...path, command); + } + }), + ); + } + + await exploreHelpPage(); + + helpPages.sort((a, b) => a.path.join(' ').localeCompare(b.path.join(' '))); + + return helpPages; +} + +// The API model for a CLI entry point +interface CliModel { + name: string; + helpPages: CliHelpPage[]; +} + +function generateCliReport(name: string, models: CliModel[]): string { + const content = [ + `## CLI Report file for "${name}"`, + '', + '> Do not edit this file. It is a report generated by `yarn build:api-reports`', + '', + ]; + + for (const model of models) { + for (const helpPage of model.helpPages) { + content.push( + `### \`${[model.name, ...helpPage.path].join(' ')}\``, + '', + '```', + `Usage: ${helpPage.usage ?? ''}`, + ); + + if (helpPage.options.length > 0) { + content.push('', 'Options:', ...helpPage.options.map(l => ` ${l}`)); + } + + if (helpPage.commands.length > 0) { + content.push('', 'Commands:', ...helpPage.commands.map(l => ` ${l}`)); + } + content.push('```', ''); + } + } + + return content.join('\n'); +} + +interface CliExtractionOptions { + projectRoot: string; + packageDirs: string[]; + isLocalBuild: boolean; +} + +export async function runCliExtraction({ + projectRoot, + packageDirs, + isLocalBuild, +}: CliExtractionOptions) { + for (const packageDir of packageDirs) { + console.log(`## Processing ${packageDir}`); + const fullDir = resolvePath(projectRoot, packageDir); + const pkgJson = await fs.readJson(resolvePath(fullDir, 'package.json')); + + if (!pkgJson.bin) { + throw new Error(`CLI Package in ${packageDir} has no bin field`); + } + + const models = new Array(); + if (typeof pkgJson.bin === 'string') { + const run = createBinRunner(fullDir, pkgJson.bin); + const helpPages = await exploreCliHelpPages(run); + models.push({ name: basename(pkgJson.bin), helpPages }); + } else { + for (const [name, path] of Object.entries(pkgJson.bin)) { + const run = createBinRunner(fullDir, path); + const helpPages = await exploreCliHelpPages(run); + models.push({ name, helpPages }); + } + } + + const report = generateCliReport(pkgJson.name, models); + + const reportPath = resolvePath(fullDir, 'cli-report.md'); + const existingReport = await fs + .readFile(reportPath, 'utf8') + .catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + + if (existingReport !== report) { + if (isLocalBuild) { + console.warn(`CLI report changed for ${packageDir}`); + await fs.writeFile(reportPath, report); + } else { + logApiReportInstructions(); + + if (existingReport) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + projectRoot, + reportPath, + )}, expecting the following content:`, + ); + console.log(''); + + console.log(report); + + logApiReportInstructions(); + } + throw new Error(`CLI report changed for ${packageDir}, `); + } + } + } +} + +async function main() { + const projectRoot = resolvePath(process.cwd()); + const isCiBuild = process.argv.includes('--ci'); + const isDocsBuild = process.argv.includes('--docs'); + const runTsc = process.argv.includes('--tsc'); + + const selectedPackageDirs = await findSpecificPackageDirs( + process.argv.slice(2).filter(arg => !arg.startsWith('--')), + ); + if (selectedPackageDirs && isCiBuild) { + throw new Error( + 'Package path arguments are not supported together with the --ci flag', + ); + } + if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { + console.log(''); + console.log( + 'TIP: You can generate api-reports for select packages by passing package paths:', + ); + console.log(''); + console.log( + ' yarn build:api-reports packages/config packages/core-plugin-api', + ); + console.log(''); + } + + let temporaryTsConfigPath: string | undefined; + if (selectedPackageDirs) { + temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); + } + const tsconfigFilePath = + temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); + + if (runTsc) { + await fs.remove(resolvePath(projectRoot, 'dist-types')); + const { status } = spawnSync( + 'yarn', + [ + 'tsc', + ['--project', tsconfigFilePath], + ['--skipLibCheck', 'false'], + ['--incremental', 'false'], + ].flat(), + { + stdio: 'inherit', + shell: true, + cwd: projectRoot, + }, + ); + if (status !== 0) { + process.exit(status); + } + } + + const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); + + const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( + projectRoot, + packageDirs, + ); + + if (tsPackageDirs.length > 0) { + console.log('# Generating package API reports'); + await runApiExtraction({ + packageDirs: tsPackageDirs, + outputDir: tmpDir, + isLocalBuild: !isCiBuild, + tsconfigFilePath, + }); + } + if (cliPackageDirs.length > 0) { + console.log('# Generating package CLI reports'); + await runCliExtraction({ + projectRoot, + packageDirs: cliPackageDirs, + isLocalBuild: !isCiBuild, + }); + } + + if (isDocsBuild) { + console.log('# Generating package documentation'); + await buildDocs({ + inputDir: tmpDir, + outputDir: resolvePath(projectRoot, 'docs/reference'), + }); + } +} + +main().catch(error => { + console.error(error.stack || String(error)); + process.exit(1); +}); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts new file mode 100644 index 0000000000..69f313a0e4 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { OptionValues } from 'commander'; +import {resolve as resolvePath} from 'path'; +import fs from 'fs-extra'; +import { spawnSync } from 'child_process'; +import {resolvePackagePath} from "@backstage/backend-common" +import {findSpecificPackageDirs, createTemporaryTsConfig, findPackageDirs, categorizePackageDirs, runApiExtraction, runCliExtraction, buildDocs} from './api-extractor'; + +export default async (paths:string[], opts: OptionValues ) => { + if(opts.ci){ + process.stderr.write( + `hello ci`, + ); + } + console.log(opts, paths) + process.stderr.write( + `hello `, + ); + + const tmpDir = resolvePath('./node_modules/.cache/api-extractor'); + const projectRoot = resolvePath(process.cwd()); + console.log(`DIRNAME: ${__dirname}, ${projectRoot}`) + const isCiBuild = opts.ci; + const isDocsBuild = opts.docs; + const runTsc = opts.tsc; + + const selectedPackageDirs = await findSpecificPackageDirs(paths + ); + if (selectedPackageDirs && isCiBuild) { + throw new Error( + 'Package path arguments are not supported together with the --ci flag', + ); + } + if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { + console.log(''); + console.log( + 'TIP: You can generate api-reports for select packages by passing package paths:', + ); + console.log(''); + console.log( + ' yarn build:api-reports packages/config packages/core-plugin-api', + ); + console.log(''); + } + + let temporaryTsConfigPath: string | undefined; + if (selectedPackageDirs) { + temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); + } + const tsconfigFilePath = + temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); + + if (runTsc) { + await fs.remove(resolvePath(projectRoot, 'dist-types')); + const { status } = spawnSync( + 'yarn', + [ + 'tsc', + ['--project', tsconfigFilePath], + ['--skipLibCheck', 'false'], + ['--incremental', 'false'], + ].flat(), + { + stdio: 'inherit', + shell: true, + cwd: projectRoot, + }, + ); + if (status !== 0) { + process.exit(status || undefined); + } + } + + const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); + + const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( + projectRoot, + packageDirs, + ); + + if (tsPackageDirs.length > 0) { + console.log('# Generating package API reports'); + await runApiExtraction({ + packageDirs: tsPackageDirs, + outputDir: tmpDir, + isLocalBuild: !isCiBuild, + tsconfigFilePath, + }); + } + if (cliPackageDirs.length > 0) { + console.log('# Generating package CLI reports'); + await runCliExtraction({ + projectRoot, + packageDirs: cliPackageDirs, + isLocalBuild: !isCiBuild, + }); + } + + if (isDocsBuild) { + console.log('# Generating package documentation'); + await buildDocs({ + inputDir: tmpDir, + outputDir: resolvePath(projectRoot, 'docs/reference'), + }); + } +}; diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts new file mode 100644 index 0000000000..8386814a68 --- /dev/null +++ b/packages/repo-tools/src/commands/index.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { Command } from 'commander'; +import { exitWithError } from '../lib/errors'; + +const configOption = [ + '--config ', + 'Config files to load instead of app-config.yaml', + (opt: string, opts: string[]) => (opts ? [...opts, opt] : [opt]), + Array(), +] as const; + +export function registerCommands(program: Command) { + program + .command('api-reports [path...]') + .option('--ci', 'Do not require environment variables to be set') + .option('--tsc', 'Only validate the frontend configuration') + .option('--docs', 'Output deprecated configuration settings') + .description('Generate an API report for changed packages') + .action( + lazy(() => import('./api-reports/api-reports').then(m => m.default)), + ); +} + +// Wraps an action function so that it always exits and handles errors +function lazy( + getActionFunc: () => Promise<(...args: any[]) => Promise>, +): (...args: any[]) => Promise { + return async (...args: any[]) => { + try { + const actionFunc = await getActionFunc(); + await actionFunc(...args); + + process.exit(0); + } catch (error) { + assertError(error); + exitWithError(error); + } + }; +} diff --git a/packages/repo-tools/src/index.ts b/packages/repo-tools/src/index.ts new file mode 100644 index 0000000000..b8e1efdf1e --- /dev/null +++ b/packages/repo-tools/src/index.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * CLI for developing Backstage plugins and apps + * + * @packageDocumentation + */ + +import { program } from 'commander'; +import chalk from 'chalk'; +import { exitWithError } from './lib/errors'; +// import { version } from './lib/version'; +import { registerCommands } from './commands'; + +const main = (argv: string[]) => { + program.name('backstage-repo-tools').version('1.0'); + + registerCommands(program); + + program.on('command:*', () => { + console.log(); + console.log(chalk.red(`Invalid command: ${program.args.join(' ')}`)); + console.log(); + program.outputHelp(); + process.exit(1); + }); + + program.parse(argv); +}; + +process.on('unhandledRejection', rejection => { + if (rejection instanceof Error) { + exitWithError(rejection); + } else { + exitWithError(new Error(`Unknown rejection: '${rejection}'`)); + } +}); + +main(process.argv); diff --git a/packages/repo-tools/src/lib/errors.ts b/packages/repo-tools/src/lib/errors.ts new file mode 100644 index 0000000000..dd3955a10a --- /dev/null +++ b/packages/repo-tools/src/lib/errors.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import chalk from 'chalk'; + +export class CustomError extends Error { + get name(): string { + return this.constructor.name; + } +} + +export class ExitCodeError extends CustomError { + readonly code: number; + + constructor(code: number, command?: string) { + super( + command + ? `Command '${command}' exited with code ${code}` + : `Child exited with code ${code}`, + ); + this.code = code; + } +} + +export function exitWithError(error: Error): never { + if (error instanceof ExitCodeError) { + process.stderr.write(`\n${chalk.red(error.message)}\n\n`); + process.exit(error.code); + } else { + process.stderr.write(`\n${chalk.red(`${error}`)}\n\n`); + process.exit(1); + } +} + +export class NotFoundError extends CustomError {} diff --git a/packages/repo-tools/src/tests/index.test.ts b/packages/repo-tools/src/tests/index.test.ts new file mode 100644 index 0000000000..b61d59e88d --- /dev/null +++ b/packages/repo-tools/src/tests/index.test.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/yarn.lock b/yarn.lock index 2793196219..5908641c91 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7653,6 +7653,20 @@ __metadata: languageName: unknown linkType: soft +"@backstage/repo-tools@workspace:*, @backstage/repo-tools@workspace:packages/repo-tools": + version: 0.0.0-use.local + resolution: "@backstage/repo-tools@workspace:packages/repo-tools" + dependencies: + "@backstage/errors": "workspace:^" + chalk: ^4.0.0 + commander: ^9.1.0 + fs-extra: 10.1.0 + ts-node: ^10.0.0 + bin: + backstage-repo-tools: bin/backstage-repo-tools + languageName: unknown + linkType: soft + "@backstage/test-utils@workspace:^, @backstage/test-utils@workspace:packages/test-utils": version: 0.0.0-use.local resolution: "@backstage/test-utils@workspace:packages/test-utils" @@ -33054,6 +33068,7 @@ __metadata: "@backstage/codemods": "workspace:*" "@backstage/create-app": "workspace:*" "@backstage/errors": "workspace:^" + "@backstage/repo-tools": "workspace:*" "@changesets/cli": ^2.14.0 "@manypkg/get-packages": ^1.1.3 "@microsoft/api-documenter": ^7.17.11 From 3bc9319895cb4e234f1e14351db2a8da068546ab Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Wed, 16 Nov 2022 15:12:38 +0100 Subject: [PATCH 2/6] Fix typing and dependencies Co-authored-by: djamaile Signed-off-by: Juan Pablo Garcia Ripa --- package.json | 9 +- packages/repo-tools/cli-report.md | 29 + packages/repo-tools/package.json | 4 + .../src/commands/api-reports/api-extractor.ts | 162 +- .../src/commands/api-reports/api-reports.ts | 191 +-- packages/repo-tools/src/commands/index.ts | 15 +- packages/repo-tools/src/tests/index.test.ts | 15 - scripts/api-extractor.ts | 1345 ----------------- yarn.lock | 15 +- 9 files changed, 190 insertions(+), 1595 deletions(-) create mode 100644 packages/repo-tools/cli-report.md delete mode 100644 packages/repo-tools/src/tests/index.test.ts delete mode 100644 scripts/api-extractor.ts diff --git a/package.json b/package.json index 551a77a614..eaba802ba0 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,8 @@ "build:backend": "yarn workspace backend build", "build:all": "backstage-cli repo build --all", "build": "backstage-cli repo build --all", - "build:api-reports-tooling": "backstage-repo-tools api-reports", "build:api-reports": "yarn build:api-reports:only --tsc", - "build:api-reports:only": "ts-node -T -P scripts/tsconfig.json scripts/api-extractor.ts", + "build:api-reports:only": "backstage-repo-tools api-reports", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs", "tsc": "tsc", "tsc:full": "backstage-cli repo clean && tsc --skipLibCheck false --incremental false", @@ -51,11 +50,7 @@ "version": "1.8.0", "dependencies": { "@backstage/errors": "workspace:^", - "@manypkg/get-packages": "^1.1.3", - "@microsoft/api-documenter": "^7.17.11", - "@microsoft/api-extractor": "^7.23.0", - "@microsoft/api-extractor-model": "^7.17.2", - "@microsoft/tsdoc": "^0.14.1" + "@manypkg/get-packages": "^1.1.3" }, "devDependencies": { "@backstage/cli": "workspace:*", diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md new file mode 100644 index 0000000000..08568827ea --- /dev/null +++ b/packages/repo-tools/cli-report.md @@ -0,0 +1,29 @@ +## CLI Report file for "@backstage/repo-tools" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +### `backstage-repo-tools` + +``` +Usage: backstage-repo-tools [options] [command] + +Options: + -V, --version + -h, --help + +Commands: + api-reports [options] [path...] + help [command] +``` + +### `backstage-repo-tools api-reports` + +``` +Usage: backstage-repo-tools api-reports [options] [path...] + +Options: + --ci + --tsc + --docs + -h, --help +``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 480e57b25a..fc217da9e4 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -31,6 +31,10 @@ }, "dependencies": { "@backstage/errors": "workspace:^", + "@microsoft/api-documenter": "^7.17.11", + "@microsoft/api-extractor": "^7.23.0", + "@microsoft/api-extractor-model": "^7.17.2", + "@microsoft/tsdoc": "0.14.1", "chalk": "^4.0.0", "commander": "^9.1.0", "fs-extra": "10.1.0", diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 85a9310ab4..d434aae0a6 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -24,7 +24,7 @@ import { dirname, join, } from 'path'; -import { spawnSync, execFile } from 'child_process'; +import { execFile } from 'child_process'; import prettier from 'prettier'; import fs from 'fs-extra'; import { @@ -49,11 +49,17 @@ import { import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable'; import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow'; import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading'; -import { CustomMarkdownEmitter } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; +import { + CustomMarkdownEmitter, + ICustomMarkdownEmitterOptions, +} from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; -const tmpDir = resolvePath(process.cwd(), './node_modules/.cache/api-extractor'); +const tmpDir = resolvePath( + process.cwd(), + './node_modules/.cache/api-extractor', +); /** * All of this monkey patching below is because MUI has these bare package.json file as a method @@ -99,7 +105,9 @@ function patchFileMessageFetcher( } = router; router.fetchAssociatedMessagesForReviewFile = - function patchedFetchAssociatedMessagesForReviewFile(ast) { + function patchedFetchAssociatedMessagesForReviewFile( + ast: AstDeclaration | undefined, + ) { const messages = fetchAssociatedMessagesForReviewFile.call(this, ast); return transform(messages, ast); }; @@ -113,7 +121,10 @@ function patchFileMessageFetcher( const originalGenerateReviewFileContent = ApiReportGenerator.generateReviewFileContent; ApiReportGenerator.generateReviewFileContent = - function decoratedGenerateReviewFileContent(collector, ...moreArgs) { + function decoratedGenerateReviewFileContent( + collector: { program: Program; messageRouter: any }, + ...moreArgs: any[] + ) { const program = collector.program as Program; // The purpose of this override is to allow the @ignore tag to be used to ignore warnings @@ -137,7 +148,9 @@ ApiReportGenerator.generateReviewFileContent = } const [, symbolName] = symbolMatch; - const sourceFile = program.getSourceFile(message.sourceFilePath); + const sourceFile = + message.sourceFilePath && + program.getSourceFile(message.sourceFilePath); if (!sourceFile) { throw new Error( `Failed to find source file in program at path "${message.sourceFilePath}"`, @@ -358,7 +371,10 @@ export async function runApiExtraction({ await fs.remove(outputDir); const entryPoints = packageDirs.map(packageDir => { - return resolvePath(process.cwd(), `./dist-types/${packageDir}/src/index.d.ts`); + return resolvePath( + process.cwd(), + `./dist-types/${packageDir}/src/index.d.ts`, + ); }); let compilerState: CompilerState | undefined = undefined; @@ -368,7 +384,11 @@ export async function runApiExtraction({ for (const packageDir of packageDirs) { console.log(`## Processing ${packageDir}`); const projectFolder = resolvePath(process.cwd(), packageDir); - const packageFolder = resolvePath(process.cwd(), './dist-types', packageDir); + const packageFolder = resolvePath( + process.cwd(), + './dist-types', + packageDir, + ); const warningCountBefore = await countApiReportWarnings(projectFolder); @@ -618,7 +638,7 @@ class ApiModelTransforms { if (serialized.members.length !== 1) { throw new Error( `Unexpected members in serialized ApiPackage, [${serialized.members - .map(m => m.kind) + .map((m: { kind: any }) => m.kind) .join(' ')}]`, ); } @@ -634,7 +654,7 @@ class ApiModelTransforms { members: [ { ...entryPoint, - members: entryPoint.members.map(member => + members: entryPoint.members.map((member: any) => transforms.reduce((m, t) => t(m), member), ), }, @@ -847,7 +867,11 @@ export async function buildDocs({ } /** @override */ - emit(stringBuilder, docNode, options) { + emit( + stringBuilder: any, + docNode: DocNode, + options: ICustomMarkdownEmitterOptions, + ) { // Hack to get rid of the leading comment of each file, since // we want the front matter to come first stringBuilder._chunks.length = 0; @@ -886,7 +910,7 @@ export async function buildDocs({ // so we hook in wherever we can. In this case we add the front matter // just before writing the breadcrumbs at the top. /** @override */ - _writeBreadcrumb(output, apiItem) { + _writeBreadcrumb(output: any, apiItem: ApiItem & { name: string }) { let title; let description; @@ -897,9 +921,12 @@ export async function buildDocs({ } else if (apiItem.kind === 'Model') { title = 'Package Index'; description = 'Index of all Backstage Packages'; - } else { + } else if (apiItem.name) { title = apiItem.name; description = `API Reference for ${apiItem.name}`; + } else { + title = apiItem.displayName; + description = `API Reference for ${apiItem.displayName}`; } // Add our front matter @@ -925,7 +952,10 @@ export async function buildDocs({ }; } - _writeModelTable(output, apiModel): void { + _writeModelTable( + output: { appendNode: (arg0: DocTable | DocHeading) => void }, + apiModel: { members: any }, + ): void { const configuration = this._tsdocConfiguration; const packagesTable = new DocTable({ @@ -998,7 +1028,10 @@ export async function buildDocs({ documenter.generateFiles(); } -export async function categorizePackageDirs(projectRoot, packageDirs) { +export async function categorizePackageDirs( + projectRoot: string, + packageDirs: any[], +) { const dirs = packageDirs.slice(); const tsPackageDirs = new Array(); const cliPackageDirs = new Array(); @@ -1086,11 +1119,11 @@ function parseHelpPage(helpPageContent: string) { // Trim away documentation const sectionItems = sectionLines .map(line => line.match(/^\s{1,8}(.*?)\s\s+/)?.[1]) - .filter(Boolean); + .filter(Boolean) as string[]; - if (sectionName.toLocaleLowerCase('en-US') === 'options:') { + if (sectionName?.toLocaleLowerCase('en-US') === 'options:') { options = sectionItems; - } else if (sectionName.toLocaleLowerCase('en-US') === 'commands:') { + } else if (sectionName?.toLocaleLowerCase('en-US') === 'commands:') { commands = sectionItems; } else { throw new Error(`Unknown CLI section: ${sectionName}`); @@ -1250,96 +1283,3 @@ export async function runCliExtraction({ } } } - -async function main() { - const projectRoot = resolvePath(process.cwd()); - const isCiBuild = process.argv.includes('--ci'); - const isDocsBuild = process.argv.includes('--docs'); - const runTsc = process.argv.includes('--tsc'); - - const selectedPackageDirs = await findSpecificPackageDirs( - process.argv.slice(2).filter(arg => !arg.startsWith('--')), - ); - if (selectedPackageDirs && isCiBuild) { - throw new Error( - 'Package path arguments are not supported together with the --ci flag', - ); - } - if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { - console.log(''); - console.log( - 'TIP: You can generate api-reports for select packages by passing package paths:', - ); - console.log(''); - console.log( - ' yarn build:api-reports packages/config packages/core-plugin-api', - ); - console.log(''); - } - - let temporaryTsConfigPath: string | undefined; - if (selectedPackageDirs) { - temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); - } - const tsconfigFilePath = - temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); - - if (runTsc) { - await fs.remove(resolvePath(projectRoot, 'dist-types')); - const { status } = spawnSync( - 'yarn', - [ - 'tsc', - ['--project', tsconfigFilePath], - ['--skipLibCheck', 'false'], - ['--incremental', 'false'], - ].flat(), - { - stdio: 'inherit', - shell: true, - cwd: projectRoot, - }, - ); - if (status !== 0) { - process.exit(status); - } - } - - const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); - - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( - projectRoot, - packageDirs, - ); - - if (tsPackageDirs.length > 0) { - console.log('# Generating package API reports'); - await runApiExtraction({ - packageDirs: tsPackageDirs, - outputDir: tmpDir, - isLocalBuild: !isCiBuild, - tsconfigFilePath, - }); - } - if (cliPackageDirs.length > 0) { - console.log('# Generating package CLI reports'); - await runCliExtraction({ - projectRoot, - packageDirs: cliPackageDirs, - isLocalBuild: !isCiBuild, - }); - } - - if (isDocsBuild) { - console.log('# Generating package documentation'); - await buildDocs({ - inputDir: tmpDir, - outputDir: resolvePath(projectRoot, 'docs/reference'), - }); - } -} - -main().catch(error => { - console.error(error.stack || String(error)); - process.exit(1); -}); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 69f313a0e4..1ae1f3edb9 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -18,104 +18,105 @@ import { OptionValues } from 'commander'; import {resolve as resolvePath} from 'path'; import fs from 'fs-extra'; import { spawnSync } from 'child_process'; -import {resolvePackagePath} from "@backstage/backend-common" -import {findSpecificPackageDirs, createTemporaryTsConfig, findPackageDirs, categorizePackageDirs, runApiExtraction, runCliExtraction, buildDocs} from './api-extractor'; +import { + findSpecificPackageDirs, + createTemporaryTsConfig, + findPackageDirs, + categorizePackageDirs, + runApiExtraction, + runCliExtraction, + buildDocs, +} from './api-extractor'; -export default async (paths:string[], opts: OptionValues ) => { - if(opts.ci){ - process.stderr.write( - `hello ci`, - ); - } - console.log(opts, paths) - process.stderr.write( - `hello `, - ); +export default async (paths: string[], opts: OptionValues) => { + const tmpDir = resolvePath( + process.cwd(), + './node_modules/.cache/api-extractor', + ); + const projectRoot = resolvePath(process.cwd()); + const isCiBuild = opts.ci; + const isDocsBuild = opts.docs; + const runTsc = opts.tsc; - const tmpDir = resolvePath('./node_modules/.cache/api-extractor'); - const projectRoot = resolvePath(process.cwd()); - console.log(`DIRNAME: ${__dirname}, ${projectRoot}`) - const isCiBuild = opts.ci; - const isDocsBuild = opts.docs; - const runTsc = opts.tsc; - - const selectedPackageDirs = await findSpecificPackageDirs(paths + const selectedPackageDirs = await findSpecificPackageDirs(paths); + + if (selectedPackageDirs && isCiBuild) { + throw new Error( + 'Package path arguments are not supported together with the --ci flag', ); - if (selectedPackageDirs && isCiBuild) { - throw new Error( - 'Package path arguments are not supported together with the --ci flag', - ); + } + if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { + console.log(''); + console.log( + 'TIP: You can generate api-reports for select packages by passing package paths:', + ); + console.log(''); + console.log( + ' yarn build:api-reports packages/config packages/core-plugin-api', + ); + console.log(''); + } + + let temporaryTsConfigPath: string | undefined; + if (selectedPackageDirs) { + temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); + } + const tsconfigFilePath = + temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); + + if (runTsc) { + await fs.remove(resolvePath(projectRoot, 'dist-types')); + const { status } = spawnSync( + 'yarn', + [ + 'tsc', + ['--project', tsconfigFilePath], + ['--skipLibCheck', 'false'], + ['--incremental', 'false'], + ].flat(), + { + stdio: 'inherit', + shell: true, + cwd: projectRoot, + }, + ); + if (status !== 0) { + process.exit(status || undefined); } - if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { - console.log(''); - console.log( - 'TIP: You can generate api-reports for select packages by passing package paths:', - ); - console.log(''); - console.log( - ' yarn build:api-reports packages/config packages/core-plugin-api', - ); - console.log(''); - } - - let temporaryTsConfigPath: string | undefined; - if (selectedPackageDirs) { - temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); - } - const tsconfigFilePath = - temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); - - if (runTsc) { - await fs.remove(resolvePath(projectRoot, 'dist-types')); - const { status } = spawnSync( - 'yarn', - [ - 'tsc', - ['--project', tsconfigFilePath], - ['--skipLibCheck', 'false'], - ['--incremental', 'false'], - ].flat(), - { - stdio: 'inherit', - shell: true, - cwd: projectRoot, - }, - ); - if (status !== 0) { - process.exit(status || undefined); - } - } - - const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); - - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( + } + + const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); + + const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( + projectRoot, + packageDirs, + ); + + console.log({ tsPackageDirs, cliPackageDirs, isDocsBuild }); + + if (tsPackageDirs.length > 0) { + console.log('# Generating package API reports'); + await runApiExtraction({ + packageDirs: tsPackageDirs, + outputDir: tmpDir, + isLocalBuild: !isCiBuild, + tsconfigFilePath, + }); + } + if (cliPackageDirs.length > 0) { + console.log('# Generating package CLI reports'); + await runCliExtraction({ projectRoot, - packageDirs, - ); - - if (tsPackageDirs.length > 0) { - console.log('# Generating package API reports'); - await runApiExtraction({ - packageDirs: tsPackageDirs, - outputDir: tmpDir, - isLocalBuild: !isCiBuild, - tsconfigFilePath, - }); - } - if (cliPackageDirs.length > 0) { - console.log('# Generating package CLI reports'); - await runCliExtraction({ - projectRoot, - packageDirs: cliPackageDirs, - isLocalBuild: !isCiBuild, - }); - } - - if (isDocsBuild) { - console.log('# Generating package documentation'); - await buildDocs({ - inputDir: tmpDir, - outputDir: resolvePath(projectRoot, 'docs/reference'), - }); - } + packageDirs: cliPackageDirs, + isLocalBuild: !isCiBuild, + }); + } + + if (isDocsBuild) { + console.log('# Generating package documentation'); + await buildDocs({ + inputDir: tmpDir, + outputDir: resolvePath(projectRoot, 'docs/reference'), + }); + } }; diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index 8386814a68..c4281b0c4f 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -18,20 +18,13 @@ import { assertError } from '@backstage/errors'; import { Command } from 'commander'; import { exitWithError } from '../lib/errors'; -const configOption = [ - '--config ', - 'Config files to load instead of app-config.yaml', - (opt: string, opts: string[]) => (opts ? [...opts, opt] : [opt]), - Array(), -] as const; - export function registerCommands(program: Command) { program .command('api-reports [path...]') - .option('--ci', 'Do not require environment variables to be set') - .option('--tsc', 'Only validate the frontend configuration') - .option('--docs', 'Output deprecated configuration settings') - .description('Generate an API report for changed packages') + .option('--ci', 'CI run checks that there is no changes on API reports') + .option('--tsc', 'executes the tsc compilation before extracting the APIs') + .option('--docs', 'generates the api documentation') + .description('Generate an API report for selected packages') .action( lazy(() => import('./api-reports/api-reports').then(m => m.default)), ); diff --git a/packages/repo-tools/src/tests/index.test.ts b/packages/repo-tools/src/tests/index.test.ts deleted file mode 100644 index b61d59e88d..0000000000 --- a/packages/repo-tools/src/tests/index.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts deleted file mode 100644 index 244008584f..0000000000 --- a/scripts/api-extractor.ts +++ /dev/null @@ -1,1345 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable import/no-extraneous-dependencies */ -/* eslint-disable no-restricted-imports */ - -import { - resolve as resolvePath, - relative as relativePath, - basename, - dirname, - join, -} from 'path'; -import { spawnSync, execFile } from 'child_process'; -import prettier from 'prettier'; -import fs from 'fs-extra'; -import { - Extractor, - ExtractorConfig, - CompilerState, - ExtractorLogLevel, - ExtractorMessage, -} from '@microsoft/api-extractor'; -import { Program } from 'typescript'; -import { - DocNode, - IDocNodeContainerParameters, - TSDocTagSyntaxKind, -} from '@microsoft/tsdoc'; -import { TSDocConfigFile } from '@microsoft/tsdoc-config'; -import { ApiPackage, ApiModel, ApiItem } from '@microsoft/api-extractor-model'; -import { - IMarkdownDocumenterOptions, - MarkdownDocumenter, -} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter'; -import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable'; -import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow'; -import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading'; -import { CustomMarkdownEmitter } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; -import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; -import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; - -const tmpDir = resolvePath(__dirname, '../node_modules/.cache/api-extractor'); - -/** - * All of this monkey patching below is because MUI has these bare package.json file as a method - * for making TypeScript accept imports like `@material-ui/core/Button`, and improve tree-shaking - * by declaring them side effect free. - * - * The package.json lookup logic in api-extractor really doesn't like that though, as it enforces - * that the 'name' field exists in all package.json files that it discovers. This below is just - * making sure that we ignore those file package.json files instead of crashing. - */ -const { - PackageJsonLookup, -} = require('@rushstack/node-core-library/lib/PackageJsonLookup'); - -const old = PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor; -PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = - function tryGetPackageJsonFilePathForPatch(path: string) { - if ( - path.includes('@material-ui') && - !dirname(path).endsWith('@material-ui') - ) { - return undefined; - } - return old.call(this, path); - }; - -/** - * Another monkey patch where we apply prettier to the API reports. This has to be patched into - * the middle of the process as API Extractor does a comparison of the contents of the old - * and new files during generation. This inserts the formatting just before that comparison. - */ -const { - ApiReportGenerator, -} = require('@microsoft/api-extractor/lib/generators/ApiReportGenerator'); - -function patchFileMessageFetcher( - router: any, - transform: (messages: ExtractorMessage[], ast?: AstDeclaration) => void, -) { - const { - fetchAssociatedMessagesForReviewFile, - fetchUnassociatedMessagesForReviewFile, - } = router; - - router.fetchAssociatedMessagesForReviewFile = - function patchedFetchAssociatedMessagesForReviewFile(ast) { - const messages = fetchAssociatedMessagesForReviewFile.call(this, ast); - return transform(messages, ast); - }; - router.fetchUnassociatedMessagesForReviewFile = - function patchedFetchUnassociatedMessagesForReviewFile() { - const messages = fetchUnassociatedMessagesForReviewFile.call(this); - return transform(messages); - }; -} - -const originalGenerateReviewFileContent = - ApiReportGenerator.generateReviewFileContent; -ApiReportGenerator.generateReviewFileContent = - function decoratedGenerateReviewFileContent(collector, ...moreArgs) { - const program = collector.program as Program; - - // The purpose of this override is to allow the @ignore tag to be used to ignore warnings - // of the form "Warning: (ae-forgotten-export) The symbol "FooBar" needs to be exported by the entry point index.d.ts" - patchFileMessageFetcher( - collector.messageRouter, - (messages: ExtractorMessage[]) => { - return messages.filter(message => { - if (message.messageId !== 'ae-forgotten-export') { - return true; - } - - // Symbol name has to be extracted from the message :( - // There's frequently no AST for these exports because type literals - // aren't traversed by the generator. - const symbolMatch = message.text.match(/The symbol "([^"]+)"/); - if (!symbolMatch) { - throw new Error( - `Failed to extract symbol name from message "${message.text}"`, - ); - } - const [, symbolName] = symbolMatch; - - const sourceFile = program.getSourceFile(message.sourceFilePath); - if (!sourceFile) { - throw new Error( - `Failed to find source file in program at path "${message.sourceFilePath}"`, - ); - } - - // The local name of the symbol within the file, rather than the exported name - const localName = (sourceFile as any).identifiers?.get(symbolName); - if (!localName) { - throw new Error( - `Unable to find local name of "${symbolName}" in ${sourceFile.fileName}`, - ); - } - - // The local AST node of the export that we're missing - const local = (sourceFile as any).locals?.get(localName); - if (!local) { - return true; - } - - // Use the type checker to look up the actual declaration(s) rather than the one in the local file - const type = program.getTypeChecker().getDeclaredTypeOfSymbol(local); - if (!type) { - throw new Error( - `Unable to find type declaration of "${symbolName}" in ${sourceFile.fileName}`, - ); - } - const declarations = type.aliasSymbol?.declarations; - if (!declarations || declarations.length === 0) { - return true; - } - - // If any of the TSDoc comments contain a @ignore tag, we ignore this message - const isIgnored = declarations.some(declaration => { - const tags = [(declaration as any).jsDoc] - .flat() - .filter(Boolean) - .flatMap((tagNode: any) => tagNode.tags); - - return tags.some(tag => tag?.tagName.text === 'ignore'); - }); - - return !isIgnored; - }); - }, - ); - - const content = originalGenerateReviewFileContent.call( - this, - collector, - ...moreArgs, - ); - return prettier.format(content, { - ...require('@spotify/prettier-config'), - parser: 'markdown', - }); - }; - -const PACKAGE_ROOTS = ['packages', 'plugins']; - -const ALLOW_WARNINGS = [ - 'packages/core-components', - 'plugins/catalog', - 'plugins/catalog-import', - 'plugins/git-release-manager', - 'plugins/jenkins', - 'plugins/kubernetes', -]; - -async function resolvePackagePath( - packagePath: string, -): Promise { - const projectRoot = resolvePath(__dirname, '..'); - const fullPackageDir = resolvePath(projectRoot, packagePath); - - const stat = await fs.stat(fullPackageDir); - if (!stat.isDirectory()) { - return undefined; - } - - try { - const packageJsonPath = join(fullPackageDir, 'package.json'); - await fs.access(packageJsonPath); - } catch (_) { - return undefined; - } - - return relativePath(projectRoot, fullPackageDir); -} - -async function findSpecificPackageDirs(unresolvedPackageDirs: string[]) { - const packageDirs = new Array(); - - for (const unresolvedPackageDir of unresolvedPackageDirs) { - const packageDir = await resolvePackagePath(unresolvedPackageDir); - if (!packageDir) { - throw new Error(`'${unresolvedPackageDir}' is not a valid package path`); - } - packageDirs.push(packageDir); - } - - if (packageDirs.length === 0) { - return undefined; - } - - return packageDirs; -} - -async function findPackageDirs() { - const packageDirs = new Array(); - const projectRoot = resolvePath(__dirname, '..'); - - for (const packageRoot of PACKAGE_ROOTS) { - const dirs = await fs.readdir(resolvePath(projectRoot, packageRoot)); - for (const dir of dirs) { - const packageDir = await resolvePackagePath(join(packageRoot, dir)); - if (!packageDir) { - continue; - } - - packageDirs.push(packageDir); - } - } - - return packageDirs; -} - -async function createTemporaryTsConfig(includedPackageDirs: string[]) { - const path = resolvePath(__dirname, '..', 'tsconfig.tmp.json'); - - process.once('exit', () => { - fs.removeSync(path); - }); - - await fs.writeJson(path, { - extends: './tsconfig.json', - include: [ - // These two contain global definitions that are needed for stable API report generation - 'packages/cli/asset-types/asset-types.d.ts', - ...includedPackageDirs.map(dir => join(dir, 'src')), - ], - }); - - return path; -} - -async function countApiReportWarnings(projectFolder: string) { - const path = resolvePath(projectFolder, 'api-report.md'); - try { - const content = await fs.readFile(path, 'utf8'); - const lines = content.split('\n'); - - const lineWarnings = lines.filter(line => - line.includes('// Warning:'), - ).length; - - const trailerStart = lines.findIndex( - line => line === '// Warnings were encountered during analysis:', - ); - const trailerWarnings = - trailerStart === -1 - ? 0 - : lines.length - - trailerStart - - 4; /* 4 lines at the trailer and after are not warnings */ - - return lineWarnings + trailerWarnings; - } catch (error) { - if (error.code === 'ENOENT') { - return 0; - } - throw error; - } -} - -async function getTsDocConfig() { - const tsdocConfigFile = await TSDocConfigFile.loadFile( - require.resolve('@microsoft/api-extractor/extends/tsdoc-base.json'), - ); - tsdocConfigFile.addTagDefinition({ - tagName: '@ignore', - syntaxKind: TSDocTagSyntaxKind.ModifierTag, - }); - tsdocConfigFile.setSupportForTag('@ignore', true); - return tsdocConfigFile; -} - -function logApiReportInstructions() { - console.log(''); - console.log( - '*************************************************************************************', - ); - console.log( - '* You have uncommitted changes to the public API of a package. *', - ); - console.log( - '* To solve this, run `yarn build:api-reports` and commit all api-report.md changes. *', - ); - console.log( - '*************************************************************************************', - ); - console.log(''); -} - -interface ApiExtractionOptions { - packageDirs: string[]; - outputDir: string; - isLocalBuild: boolean; - tsconfigFilePath: string; -} - -async function runApiExtraction({ - packageDirs, - outputDir, - isLocalBuild, - tsconfigFilePath, -}: ApiExtractionOptions) { - await fs.remove(outputDir); - - const entryPoints = packageDirs.map(packageDir => { - return resolvePath(__dirname, `../dist-types/${packageDir}/src/index.d.ts`); - }); - - let compilerState: CompilerState | undefined = undefined; - - const warnings = new Array(); - - for (const packageDir of packageDirs) { - console.log(`## Processing ${packageDir}`); - const projectFolder = resolvePath(__dirname, '..', packageDir); - const packageFolder = resolvePath(__dirname, '../dist-types', packageDir); - - const warningCountBefore = await countApiReportWarnings(projectFolder); - - const extractorConfig = ExtractorConfig.prepare({ - configObject: { - mainEntryPointFilePath: resolvePath(packageFolder, 'src/index.d.ts'), - bundledPackages: [], - - compiler: { - tsconfigFilePath, - }, - - apiReport: { - enabled: true, - reportFileName: 'api-report.md', - reportFolder: projectFolder, - reportTempFolder: resolvePath(outputDir, ''), - }, - - docModel: { - enabled: true, - apiJsonFilePath: resolvePath( - outputDir, - '.api.json', - ), - }, - - dtsRollup: { - enabled: false, - }, - - tsdocMetadata: { - enabled: false, - }, - - messages: { - // Silence compiler warnings, as these will prevent the CI build to work - compilerMessageReporting: { - default: { - logLevel: 'none' as ExtractorLogLevel.None, - // These contain absolute file paths, so can't be included in the report - // addToApiReportFile: true, - }, - }, - extractorMessageReporting: { - default: { - logLevel: 'warning' as ExtractorLogLevel.Warning, - addToApiReportFile: true, - }, - }, - tsdocMessageReporting: { - default: { - logLevel: 'warning' as ExtractorLogLevel.Warning, - addToApiReportFile: true, - }, - }, - }, - - newlineKind: 'lf', - - projectFolder, - }, - configObjectFullPath: projectFolder, - packageJsonFullPath: resolvePath(projectFolder, 'package.json'), - tsdocConfigFile: await getTsDocConfig(), - }); - - // The `packageFolder` needs to point to the location within `dist-types` in order for relative - // paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`, - // which needs to point to the actual file, so we override `packageFolder` afterwards. - ( - extractorConfig as { - packageFolder: string; - } - ).packageFolder = packageFolder; - - if (!compilerState) { - compilerState = CompilerState.create(extractorConfig, { - additionalEntryPoints: entryPoints, - }); - } - - // Message verbosity can't be configured, so just skip the check instead - (Extractor as any)._checkCompilerCompatibility = () => {}; - - let shouldLogInstructions = false; - let conflictingFile: undefined | string = undefined; - - // Invoke API Extractor - const extractorResult = Extractor.invoke(extractorConfig, { - localBuild: isLocalBuild, - showVerboseMessages: false, - showDiagnostics: false, - messageCallback(message) { - if ( - message.text.includes( - 'You have changed the public API signature for this project.', - ) - ) { - shouldLogInstructions = true; - const match = message.text.match( - /Please copy the file "(.*)" to "api-report\.md"/, - ); - if (match) { - conflictingFile = match[1]; - } - } - }, - compilerState, - }); - - if (!extractorResult.succeeded) { - if (shouldLogInstructions) { - logApiReportInstructions(); - - if (conflictingFile) { - console.log(''); - console.log( - `The conflicting file is ${relativePath( - tmpDir, - conflictingFile, - )}, with the following content:`, - ); - console.log(''); - - const content = await fs.readFile(conflictingFile, 'utf8'); - console.log(content); - - logApiReportInstructions(); - } - } - - throw new Error( - `API Extractor completed with ${extractorResult.errorCount} errors` + - ` and ${extractorResult.warningCount} warnings`, - ); - } - - const warningCountAfter = await countApiReportWarnings(projectFolder); - if (warningCountAfter > 0 && !ALLOW_WARNINGS.includes(packageDir)) { - throw new Error( - `The API Report for ${packageDir} is not allowed to have warnings`, - ); - } - if (warningCountAfter === 0 && ALLOW_WARNINGS.includes(packageDir)) { - console.log( - `No need to allow warnings for ${packageDir}, it does not have any`, - ); - } - if (warningCountAfter > warningCountBefore) { - warnings.push( - `The API Report for ${packageDir} introduces new warnings. ` + - 'Please fix these warnings in order to keep the API Reports tidy.', - ); - } - } - - if (warnings.length > 0) { - console.warn(); - for (const warning of warnings) { - console.warn(warning); - } - console.warn(); - } -} - -/* -WARNING: Bring a blanket if you're gonna read the code below - -There's some weird shit going on here, and it's because we cba -forking rushstack to modify the api-documenter markdown generation, -which otherwise is the recommended way to do customizations. -*/ - -type ExcerptToken = { - kind: string; - text: string; - canonicalReference?: string; -}; - -class ExcerptTokenMatcher { - readonly #tokens: ExcerptToken[]; - - constructor(tokens: ExcerptToken[]) { - this.#tokens = tokens.slice(); - } - - nextContent() { - const token = this.#tokens.shift(); - if (token?.kind === 'Content') { - return token.text; - } - return undefined; - } - - matchContent(expectedText: string) { - const text = this.nextContent(); - return text !== expectedText; - } - - getTokensUntilArrow() { - const tokens = []; - for (;;) { - const token = this.#tokens.shift(); - if (token === undefined) { - return undefined; - } - if (token.kind === 'Content' && token.text === ') => ') { - return tokens; - } - tokens.push(token); - } - } - - getComponentReturnTokens() { - const first = this.#tokens.shift(); - if (!first) { - return undefined; - } - const second = this.#tokens.shift(); - - if (this.#tokens.length !== 0) { - return undefined; - } - if (first.kind !== 'Reference' || first.text !== 'JSX.Element') { - return undefined; - } - if (!second) { - return [first]; - } else if (second.kind === 'Content' && second.text === ' | null') { - return [first, second]; - } - return undefined; - } -} - -class ApiModelTransforms { - static deserializeWithTransforms( - serialized: any, - transforms: Array<(member: any) => any>, - ): ApiPackage { - if (serialized.kind !== 'Package') { - throw new Error( - `Unexpected root kind in serialized ApiPackage, ${serialized.kind}`, - ); - } - if (serialized.members.length !== 1) { - throw new Error( - `Unexpected members in serialized ApiPackage, [${serialized.members - .map(m => m.kind) - .join(' ')}]`, - ); - } - const [entryPoint] = serialized.members; - if (entryPoint.kind !== 'EntryPoint') { - throw new Error( - `Unexpected kind in serialized ApiPackage member, ${entryPoint.kind}`, - ); - } - - const transformed = { - ...serialized, - members: [ - { - ...entryPoint, - members: entryPoint.members.map(member => - transforms.reduce((m, t) => t(m), member), - ), - }, - ], - }; - - return ApiPackage.deserialize( - transformed, - transformed.metadata, - ) as ApiPackage; - } - - static transformArrowComponents = (member: any) => { - if (member.kind !== 'Variable') { - return member; - } - - const { name, excerptTokens } = member; - - // First letter in name must be uppercase - const [firstChar] = name; - if (firstChar.toLocaleUpperCase('en-US') !== firstChar) { - return member; - } - - // First content must match expected declaration format - const tokens = new ExcerptTokenMatcher(excerptTokens); - if (tokens.nextContent() !== `${name}: `) { - return member; - } - - // Next needs to be an arrow with `props` parameters or no parameters - // followed by a return type of `JSX.Element | null` or just `JSX.Element` - const declStart = tokens.nextContent(); - if (declStart === '(props: ' || declStart === '(_props: ') { - const props = tokens.getTokensUntilArrow(); - const ret = tokens.getComponentReturnTokens(); - if (props && ret) { - return this.makeComponentMember(member, ret, props); - } - } else if (declStart === '() => ') { - const ret = tokens.getComponentReturnTokens(); - if (ret) { - return this.makeComponentMember(member, ret); - } - } - return member; - }; - - static makeComponentMember( - member: any, - ret: ExcerptToken[], - props?: ExcerptToken[], - ) { - const declTokens = props - ? [ - { - kind: 'Content', - text: `export declare function ${member.name}(props: `, - }, - ...props, - { - kind: 'Content', - text: '): ', - }, - ] - : [ - { - kind: 'Content', - text: `export declare function ${member.name}(): `, - }, - ]; - - return { - kind: 'Function', - name: member.name, - releaseTag: member.releaseTag, - docComment: member.docComment ?? '', - canonicalReference: member.canonicalReference, - excerptTokens: [...declTokens, ...ret], - returnTypeTokenRange: { - startIndex: declTokens.length, - endIndex: declTokens.length + ret.length, - }, - parameters: props - ? [ - { - parameterName: 'props', - parameterTypeTokenRange: { - startIndex: 1, - endIndex: 1 + props.length, - }, - }, - ] - : [], - overloadIndex: 1, - }; - } - - static transformTrimDeclare = (member: any) => { - const { excerptTokens } = member; - const firstContent = new ExcerptTokenMatcher(excerptTokens).nextContent(); - if (firstContent && firstContent.startsWith('export declare ')) { - return { - ...member, - excerptTokens: [ - { - kind: 'Content', - text: firstContent.slice('export declare '.length), - }, - ...excerptTokens.slice(1), - ], - }; - } - return member; - }; -} - -async function buildDocs({ - inputDir, - outputDir, -}: { - inputDir: string; - outputDir: string; -}) { - // We start by constructing our own model from the files so that - // we get a change to modify them, as the model is otherwise read-only. - const parseFile = async (filename: string): Promise => { - console.log(`Reading ${filename}`); - return fs.readJson(resolvePath(inputDir, filename)); - }; - - const filenames = await fs.readdir(inputDir); - const serializedPackages = await Promise.all( - filenames - .filter(filename => filename.match(/\.api\.json$/i)) - .map(parseFile), - ); - - const newModel = new ApiModel(); - for (const serialized of serializedPackages) { - newModel.addMember( - ApiModelTransforms.deserializeWithTransforms(serialized, [ - ApiModelTransforms.transformArrowComponents, - ApiModelTransforms.transformTrimDeclare, - ]), - ); - } - - // The doc AST need to be extended with custom nodes if we want to - // add any extra content. - // This one is for the YAML front matter that we need for the microsite. - class DocFrontMatter extends DocNode { - static kind = 'DocFrontMatter'; - - public readonly values: { [name: string]: unknown }; - - public constructor( - parameters: IDocNodeContainerParameters & { - values: { [name: string]: unknown }; - }, - ) { - super(parameters); - this.values = parameters.values; - } - - /** @override */ - public get kind(): string { - return DocFrontMatter.kind; - } - } - - // This is where we actually write the markdown and where we can hook - // in the rendering of our own nodes. - class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter { - // Until https://github.com/microsoft/rushstack/issues/2914 gets fixed or we change markdown renderer we need a fix - // to render pipe | character correctly. - protected getEscapedText(text: string): string { - return text - .replace(/\\/g, '\\\\') // first replace the escape character - .replace(/[*#[\]_`~]/g, x => `\\${x}`) // then escape any special characters - .replace(/---/g, '\\-\\-\\-') // hyphens only if it's 3 or more - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/\|/g, '|'); - } - /** @override */ - protected writeNode( - docNode: DocNode, - context: IMarkdownEmitterContext, - docNodeSiblings: boolean, - ): void { - switch (docNode.kind) { - case DocFrontMatter.kind: { - const node = docNode as DocFrontMatter; - context.writer.writeLine('---'); - for (const [name, value] of Object.entries(node.values)) { - if (value) { - context.writer.writeLine(`${name}: ${value}`); - } - } - context.writer.writeLine('---'); - context.writer.writeLine(); - break; - } - default: - super.writeNode(docNode, context, docNodeSiblings); - } - } - - /** @override */ - emit(stringBuilder, docNode, options) { - // Hack to get rid of the leading comment of each file, since - // we want the front matter to come first - stringBuilder._chunks.length = 0; - return super.emit(stringBuilder, docNode, options); - } - } - - class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) { - constructor(options: IMarkdownDocumenterOptions) { - super(options); - - // It's a strict model, we gotta register the allowed usage of our new node - this._tsdocConfiguration.docNodeManager.registerDocNodes( - '@backstage/docs', - [{ docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter }], - ); - this._tsdocConfiguration.docNodeManager.registerAllowableChildren( - 'Paragraph', - [DocFrontMatter.kind], - ); - - this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel); - } - - private _getFilenameForApiItem(apiItem: ApiItem): string { - const filename: string = super._getFilenameForApiItem(apiItem); - - if (filename.includes('.html.')) { - return filename.replace(/\.html\./g, '._html.'); - } - - return filename; - } - - // We don't really get many chances to modify the generated AST - // so we hook in wherever we can. In this case we add the front matter - // just before writing the breadcrumbs at the top. - /** @override */ - _writeBreadcrumb(output, apiItem) { - let title; - let description; - - const name = apiItem.getScopedNameWithinPackage(); - if (name) { - title = name; - description = `API reference for ${apiItem.getScopedNameWithinPackage()}`; - } else if (apiItem.kind === 'Model') { - title = 'Package Index'; - description = 'Index of all Backstage Packages'; - } else { - title = apiItem.name; - description = `API Reference for ${apiItem.name}`; - } - - // Add our front matter - output.appendNodeInParagraph( - new DocFrontMatter({ - configuration: this._tsdocConfiguration, - values: { - id: this._getFilenameForApiItem(apiItem).slice(0, -3), - title, - description, - }, - }), - ); - - // Now write the actual breadcrumbs - super._writeBreadcrumb(output, apiItem); - - // We wanna ignore the header that always gets written after the breadcrumb - // This otherwise becomes more or less a duplicate of the title in the front matter - const oldAppendNode = output.appendNode; - output.appendNode = () => { - output.appendNode = oldAppendNode; - }; - } - - _writeModelTable(output, apiModel): void { - const configuration = this._tsdocConfiguration; - - const packagesTable = new DocTable({ - configuration, - headerTitles: ['Package', 'Description'], - }); - - const pluginsTable = new DocTable({ - configuration, - headerTitles: ['Package', 'Description'], - }); - - for (const apiMember of apiModel.members) { - const row = new DocTableRow({ configuration }, [ - this._createTitleCell(apiMember), - this._createDescriptionCell(apiMember), - ]); - - if (apiMember.kind === 'Package') { - this._writeApiItemPage(apiMember); - - if (apiMember.name.startsWith('@backstage/plugin-')) { - pluginsTable.addRow(row); - } else { - packagesTable.addRow(row); - } - } - } - - if (packagesTable.rows.length > 0) { - output.appendNode( - new DocHeading({ - configuration: this._tsdocConfiguration, - title: 'Packages', - }), - ); - output.appendNode(packagesTable); - } - - if (pluginsTable.rows.length > 0) { - output.appendNode( - new DocHeading({ - configuration: this._tsdocConfiguration, - title: 'Plugins', - }), - ); - output.appendNode(pluginsTable); - } - } - } - - // This is root of the documentation generation, but it's not directly - // responsible for generating markdown, it just constructs an AST that - // is the consumed by an emitter to actually write the files. - const documenter = new CustomMarkdownDocumenter({ - apiModel: newModel, - documenterConfig: { - outputTarget: 'markdown', - newlineKind: '\n', - // De ba dålig kod - configFilePath: '', - configFile: {}, - } as any, - outputFolder: outputDir, - }); - - // Clean up existing stuff and write ALL the docs! - await fs.remove(outputDir); - await fs.ensureDir(outputDir); - documenter.generateFiles(); -} - -async function categorizePackageDirs(projectRoot, packageDirs) { - const dirs = packageDirs.slice(); - const tsPackageDirs = new Array(); - const cliPackageDirs = new Array(); - - await Promise.all( - Array(10) - .fill(0) - .map(async () => { - for (;;) { - const dir = dirs.pop(); - if (!dir) { - return; - } - - const pkgJson = await fs - .readJson(resolvePath(projectRoot, dir, 'package.json')) - .catch(error => { - if (error.code === 'ENOENT') { - return undefined; - } - throw error; - }); - const role = pkgJson?.backstage?.role; - if (!role) { - throw new Error(`No backstage.role in ${dir}/package.json`); - } - if (role === 'cli') { - cliPackageDirs.push(dir); - } else if (role !== 'frontend' && role !== 'backend') { - tsPackageDirs.push(dir); - } - } - }), - ); - - return { tsPackageDirs, cliPackageDirs }; -} - -function createBinRunner(cwd: string, path: string) { - return async (...command: string[]) => - new Promise((resolve, reject) => { - execFile( - 'node', - [path, ...command], - { - cwd, - shell: true, - timeout: 60000, - maxBuffer: 1024 * 1024, - }, - (err, stdout, stderr) => { - if (err) { - reject(new Error(`${err.message}\n${stderr}`)); - } else if (stderr) { - reject(new Error(`Command printed error output: ${stderr}`)); - } else { - resolve(stdout); - } - }, - ); - }); -} - -function parseHelpPage(helpPageContent: string) { - const [, usage] = helpPageContent.match(/^\s*Usage: (.*)$/im) ?? []; - const lines = helpPageContent.split(/\r?\n/); - - let options = new Array(); - let commands = new Array(); - - while (lines.length > 0) { - while (lines.length > 0 && !lines[0].endsWith(':')) { - lines.shift(); - } - if (lines.length > 0) { - // Start of a new section, e.g. "Options:" - const sectionName = lines.shift(); - // Take lines until we hit the next section or the end - const sectionEndIndex = lines.findIndex( - line => line && !line.match(/^\s/), - ); - const sectionLines = lines.slice(0, sectionEndIndex); - lines.splice(0, sectionLines.length); - - // Trim away documentation - const sectionItems = sectionLines - .map(line => line.match(/^\s{1,8}(.*?)\s\s+/)?.[1]) - .filter(Boolean); - - if (sectionName.toLocaleLowerCase('en-US') === 'options:') { - options = sectionItems; - } else if (sectionName.toLocaleLowerCase('en-US') === 'commands:') { - commands = sectionItems; - } else { - throw new Error(`Unknown CLI section: ${sectionName}`); - } - } - } - - return { - usage, - options, - commands, - }; -} - -// Represents the help page os a CLI command -interface CliHelpPage { - // Path of commands to reach this page - path: string[]; - // Parsed content - usage: string | undefined; - options: string[]; - commands: string[]; -} - -async function exploreCliHelpPages( - run: (...args: string[]) => Promise, -): Promise { - const helpPages = new Array(); - - async function exploreHelpPage(...path: string[]) { - const content = await run(...path, '--help'); - const parsed = parseHelpPage(content); - helpPages.push({ path, ...parsed }); - - await Promise.all( - parsed.commands.map(async fullCommand => { - const command = fullCommand.split(/[|\s]/)[0]; - if (command !== 'help') { - await exploreHelpPage(...path, command); - } - }), - ); - } - - await exploreHelpPage(); - - helpPages.sort((a, b) => a.path.join(' ').localeCompare(b.path.join(' '))); - - return helpPages; -} - -// The API model for a CLI entry point -interface CliModel { - name: string; - helpPages: CliHelpPage[]; -} - -function generateCliReport(name: string, models: CliModel[]): string { - const content = [ - `## CLI Report file for "${name}"`, - '', - '> Do not edit this file. It is a report generated by `yarn build:api-reports`', - '', - ]; - - for (const model of models) { - for (const helpPage of model.helpPages) { - content.push( - `### \`${[model.name, ...helpPage.path].join(' ')}\``, - '', - '```', - `Usage: ${helpPage.usage ?? ''}`, - ); - - if (helpPage.options.length > 0) { - content.push('', 'Options:', ...helpPage.options.map(l => ` ${l}`)); - } - - if (helpPage.commands.length > 0) { - content.push('', 'Commands:', ...helpPage.commands.map(l => ` ${l}`)); - } - content.push('```', ''); - } - } - - return content.join('\n'); -} - -interface CliExtractionOptions { - projectRoot: string; - packageDirs: string[]; - isLocalBuild: boolean; -} - -async function runCliExtraction({ - projectRoot, - packageDirs, - isLocalBuild, -}: CliExtractionOptions) { - for (const packageDir of packageDirs) { - console.log(`## Processing ${packageDir}`); - const fullDir = resolvePath(projectRoot, packageDir); - const pkgJson = await fs.readJson(resolvePath(fullDir, 'package.json')); - - if (!pkgJson.bin) { - throw new Error(`CLI Package in ${packageDir} has no bin field`); - } - - const models = new Array(); - if (typeof pkgJson.bin === 'string') { - const run = createBinRunner(fullDir, pkgJson.bin); - const helpPages = await exploreCliHelpPages(run); - models.push({ name: basename(pkgJson.bin), helpPages }); - } else { - for (const [name, path] of Object.entries(pkgJson.bin)) { - const run = createBinRunner(fullDir, path); - const helpPages = await exploreCliHelpPages(run); - models.push({ name, helpPages }); - } - } - - const report = generateCliReport(pkgJson.name, models); - - const reportPath = resolvePath(fullDir, 'cli-report.md'); - const existingReport = await fs - .readFile(reportPath, 'utf8') - .catch(error => { - if (error.code === 'ENOENT') { - return undefined; - } - throw error; - }); - - if (existingReport !== report) { - if (isLocalBuild) { - console.warn(`CLI report changed for ${packageDir}`); - await fs.writeFile(reportPath, report); - } else { - logApiReportInstructions(); - - if (existingReport) { - console.log(''); - console.log( - `The conflicting file is ${relativePath( - projectRoot, - reportPath, - )}, expecting the following content:`, - ); - console.log(''); - - console.log(report); - - logApiReportInstructions(); - } - throw new Error(`CLI report changed for ${packageDir}, `); - } - } - } -} - -async function main() { - const projectRoot = resolvePath(__dirname, '..'); - const isCiBuild = process.argv.includes('--ci'); - const isDocsBuild = process.argv.includes('--docs'); - const runTsc = process.argv.includes('--tsc'); - - const selectedPackageDirs = await findSpecificPackageDirs( - process.argv.slice(2).filter(arg => !arg.startsWith('--')), - ); - if (selectedPackageDirs && isCiBuild) { - throw new Error( - 'Package path arguments are not supported together with the --ci flag', - ); - } - if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { - console.log(''); - console.log( - 'TIP: You can generate api-reports for select packages by passing package paths:', - ); - console.log(''); - console.log( - ' yarn build:api-reports packages/config packages/core-plugin-api', - ); - console.log(''); - } - - let temporaryTsConfigPath: string | undefined; - if (selectedPackageDirs) { - temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); - } - const tsconfigFilePath = - temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); - - if (runTsc) { - await fs.remove(resolvePath(projectRoot, 'dist-types')); - const { status } = spawnSync( - 'yarn', - [ - 'tsc', - ['--project', tsconfigFilePath], - ['--skipLibCheck', 'false'], - ['--incremental', 'false'], - ].flat(), - { - stdio: 'inherit', - shell: true, - cwd: projectRoot, - }, - ); - if (status !== 0) { - process.exit(status); - } - } - - const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); - - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( - projectRoot, - packageDirs, - ); - - if (tsPackageDirs.length > 0) { - console.log('# Generating package API reports'); - await runApiExtraction({ - packageDirs: tsPackageDirs, - outputDir: tmpDir, - isLocalBuild: !isCiBuild, - tsconfigFilePath, - }); - } - if (cliPackageDirs.length > 0) { - console.log('# Generating package CLI reports'); - await runCliExtraction({ - projectRoot, - packageDirs: cliPackageDirs, - isLocalBuild: !isCiBuild, - }); - } - - if (isDocsBuild) { - console.log('# Generating package documentation'); - await buildDocs({ - inputDir: tmpDir, - outputDir: resolvePath(projectRoot, 'docs/reference'), - }); - } -} - -main().catch(error => { - console.error(error.stack || String(error)); - process.exit(1); -}); diff --git a/yarn.lock b/yarn.lock index 5908641c91..78b203c367 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7658,6 +7658,10 @@ __metadata: resolution: "@backstage/repo-tools@workspace:packages/repo-tools" dependencies: "@backstage/errors": "workspace:^" + "@microsoft/api-documenter": ^7.17.11 + "@microsoft/api-extractor": ^7.23.0 + "@microsoft/api-extractor-model": ^7.17.2 + "@microsoft/tsdoc": 0.14.1 chalk: ^4.0.0 commander: ^9.1.0 fs-extra: 10.1.0 @@ -10501,13 +10505,6 @@ __metadata: languageName: node linkType: hard -"@microsoft/tsdoc@npm:^0.14.1": - version: 0.14.2 - resolution: "@microsoft/tsdoc@npm:0.14.2" - checksum: b167c89e916ba73ee20b9c9d5dba6aa3a0de25ed3d50050e8a344dca7cd43cb2e1059bd515c820369b6e708901dd3fda476a42bc643ca74a35671ce77f724a3a - languageName: node - linkType: hard - "@mswjs/cookies@npm:^0.2.0, @mswjs/cookies@npm:^0.2.2": version: 0.2.2 resolution: "@mswjs/cookies@npm:0.2.2" @@ -33071,10 +33068,6 @@ __metadata: "@backstage/repo-tools": "workspace:*" "@changesets/cli": ^2.14.0 "@manypkg/get-packages": ^1.1.3 - "@microsoft/api-documenter": ^7.17.11 - "@microsoft/api-extractor": ^7.23.0 - "@microsoft/api-extractor-model": ^7.17.2 - "@microsoft/tsdoc": ^0.14.1 "@octokit/rest": ^19.0.3 "@spotify/prettier-config": ^14.0.0 "@techdocs/cli": "workspace:*" From 99713fd6719a7d7c5b108057b5e13665328b0e01 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Wed, 16 Nov 2022 15:31:40 +0100 Subject: [PATCH 3/6] adding changeset Signed-off-by: Juan Pablo Garcia Ripa --- .changeset/clean-paws-fry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/clean-paws-fry.md diff --git a/.changeset/clean-paws-fry.md b/.changeset/clean-paws-fry.md new file mode 100644 index 0000000000..5e66a686c0 --- /dev/null +++ b/.changeset/clean-paws-fry.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': minor +--- + +Introducing repo-tools package From cf69f335018ed5336bf4e4a6f1228e7888c20282 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Wed, 16 Nov 2022 17:15:01 +0100 Subject: [PATCH 4/6] clean up and fix descriptions Signed-off-by: Juan Pablo Garcia Ripa --- package.json | 1 - packages/repo-tools/.eslintrc.js | 1 - packages/repo-tools/README.md | 12 ++++++++++++ ...{backstage-repo-tools.js => backstage-repo-tools} | 0 packages/repo-tools/package.json | 5 +---- .../src/commands/api-reports/api-reports.ts | 2 +- packages/repo-tools/src/index.ts | 3 +-- 7 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 packages/repo-tools/README.md rename packages/repo-tools/bin/{backstage-repo-tools.js => backstage-repo-tools} (100%) mode change 100644 => 100755 diff --git a/package.json b/package.json index eaba802ba0..d40e7dfbc0 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "start-backend": "yarn workspace example-backend start", "build:backend": "yarn workspace backend build", "build:all": "backstage-cli repo build --all", - "build": "backstage-cli repo build --all", "build:api-reports": "yarn build:api-reports:only --tsc", "build:api-reports:only": "backstage-repo-tools api-reports", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs", diff --git a/packages/repo-tools/.eslintrc.js b/packages/repo-tools/.eslintrc.js index e9ea2f5bfb..599478bcc8 100644 --- a/packages/repo-tools/.eslintrc.js +++ b/packages/repo-tools/.eslintrc.js @@ -1,5 +1,4 @@ module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { - ignorePatterns: ['templates/**'], rules: { 'no-console': 0, }, diff --git a/packages/repo-tools/README.md b/packages/repo-tools/README.md new file mode 100644 index 0000000000..e03d48d0ae --- /dev/null +++ b/packages/repo-tools/README.md @@ -0,0 +1,12 @@ +# @backstage/repo-tools + +This package provides a CLI for backstage repo tooling. + +## Installation + +Install the package via Yarn: + +```sh +yarn add @backstage/repo-tools +``` + diff --git a/packages/repo-tools/bin/backstage-repo-tools.js b/packages/repo-tools/bin/backstage-repo-tools old mode 100644 new mode 100755 similarity index 100% rename from packages/repo-tools/bin/backstage-repo-tools.js rename to packages/repo-tools/bin/backstage-repo-tools diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index fc217da9e4..bffed77cab 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/repo-tools", - "description": "", + "description": "CLI for Backstage repo tooling ", "version": "0.0.0", "publishConfig": { "access": "public" @@ -41,9 +41,6 @@ "ts-node": "^10.0.0" }, "files": [ - "asset-types", - "templates", - "config", "bin", "dist/**/*.js" ], diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 1ae1f3edb9..0efbf0ec06 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -15,7 +15,7 @@ */ import { OptionValues } from 'commander'; -import {resolve as resolvePath} from 'path'; +import { resolve as resolvePath } from 'path'; import fs from 'fs-extra'; import { spawnSync } from 'child_process'; import { diff --git a/packages/repo-tools/src/index.ts b/packages/repo-tools/src/index.ts index b8e1efdf1e..1d3cd5ecb0 100644 --- a/packages/repo-tools/src/index.ts +++ b/packages/repo-tools/src/index.ts @@ -15,7 +15,7 @@ */ /** - * CLI for developing Backstage plugins and apps + * CLI for Backstage repo tooling * * @packageDocumentation */ @@ -23,7 +23,6 @@ import { program } from 'commander'; import chalk from 'chalk'; import { exitWithError } from './lib/errors'; -// import { version } from './lib/version'; import { registerCommands } from './commands'; const main = (argv: string[]) => { From e9dc9dbd101f9780015b6c9f7238c1207793579b Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Thu, 17 Nov 2022 15:16:07 +0100 Subject: [PATCH 5/6] prettier Signed-off-by: Juan Pablo Garcia Ripa --- packages/repo-tools/README.md | 1 - packages/repo-tools/src/index.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/repo-tools/README.md b/packages/repo-tools/README.md index e03d48d0ae..eb9f70dd72 100644 --- a/packages/repo-tools/README.md +++ b/packages/repo-tools/README.md @@ -9,4 +9,3 @@ Install the package via Yarn: ```sh yarn add @backstage/repo-tools ``` - diff --git a/packages/repo-tools/src/index.ts b/packages/repo-tools/src/index.ts index 1d3cd5ecb0..ed0e2992d3 100644 --- a/packages/repo-tools/src/index.ts +++ b/packages/repo-tools/src/index.ts @@ -15,7 +15,7 @@ */ /** - * CLI for Backstage repo tooling + * CLI for Backstage repo tooling * * @packageDocumentation */ From 7ae29d87304c2c88712064250ea9b44f375eb990 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Fri, 18 Nov 2022 11:38:16 +0100 Subject: [PATCH 6/6] remove debuge console.log Signed-off-by: Juan Pablo Garcia Ripa --- packages/repo-tools/src/commands/api-reports/api-reports.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 0efbf0ec06..3ac740d596 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -92,8 +92,6 @@ export default async (paths: string[], opts: OptionValues) => { packageDirs, ); - console.log({ tsPackageDirs, cliPackageDirs, isDocsBuild }); - if (tsPackageDirs.length > 0) { console.log('# Generating package API reports'); await runApiExtraction({