diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index e04f82dfd9..c5d812e4a4 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -14,269 +14,8 @@ * limitations under the License. */ -import { groupBy } from 'lodash'; -import { join, relative as relativePath, resolve as resolvePath } from 'path'; import fs from 'fs-extra'; -import { - CompilerState, - Extractor, - ExtractorConfig, - ExtractorLogLevel, - ExtractorMessage, -} from '@microsoft/api-extractor'; -import { Program } from 'typescript'; -import { - DocBlockTag, - DocLinkTag, - DocNode, - DocPlainText, - IDocNodeContainerParameters, - Standardization, - TSDocConfiguration, - TSDocTagSyntaxKind, -} from '@microsoft/tsdoc'; -import { TSDocConfigFile } from '@microsoft/tsdoc-config'; -import { - ApiItem, - ApiItemKind, - ApiModel, - ApiPackage, -} from '@microsoft/api-extractor-model'; -import { - IMarkdownDocumenterOptions, - MarkdownDocumenter, -} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter'; -import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable'; -import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow'; -import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading'; -import { - CustomMarkdownEmitter, - ICustomMarkdownEmitterOptions, -} from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; -import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; -import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; import { paths as cliPaths } from '../../lib/paths'; -import { minimatch } from 'minimatch'; -import { getPackageExportDetails } from '../../lib/getPackageExportDetails'; - -const tmpDir = cliPaths.resolveTargetRoot( - './node_modules/.cache/api-extractor', -); - -/** - * All of this monkey patching below is for apply prettier to the API reports. This has to be patched into - * the middle of the process as API Extractor does a comparison of the contents of the old - * and new files during generation. This inserts the formatting just before that comparison. - */ -const { - ApiReportGenerator, -} = require('@microsoft/api-extractor/lib/generators/ApiReportGenerator'); - -function patchFileMessageFetcher( - router: any, - transform: (messages: ExtractorMessage[], ast?: AstDeclaration) => void, -) { - const { - fetchAssociatedMessagesForReviewFile, - fetchUnassociatedMessagesForReviewFile, - } = router; - - router.fetchAssociatedMessagesForReviewFile = - function patchedFetchAssociatedMessagesForReviewFile( - ast: AstDeclaration | undefined, - ) { - const messages = fetchAssociatedMessagesForReviewFile.call(this, ast); - return transform(messages, ast); - }; - router.fetchUnassociatedMessagesForReviewFile = - function patchedFetchUnassociatedMessagesForReviewFile() { - const messages = fetchUnassociatedMessagesForReviewFile.call(this); - return transform(messages); - }; -} - -const originalGenerateReviewFileContent = - ApiReportGenerator.generateReviewFileContent; -ApiReportGenerator.generateReviewFileContent = - function decoratedGenerateReviewFileContent( - collector: { program: Program; messageRouter: any }, - ...moreArgs: any[] - ) { - const program = collector.program as Program; - // The purpose of this override is to allow the @ignore tag to be used to ignore warnings - // of the form "Warning: (ae-forgotten-export) The symbol "FooBar" needs to be exported by the entry point index.d.ts" - patchFileMessageFetcher( - collector.messageRouter, - (messages: ExtractorMessage[]) => { - return messages.filter(message => { - if (message.messageId !== 'ae-forgotten-export') { - return true; - } - - // Symbol name has to be extracted from the message :( - // There's frequently no AST for these exports because type literals - // aren't traversed by the generator. - const symbolMatch = message.text.match(/The symbol "([^"]+)"/); - if (!symbolMatch) { - throw new Error( - `Failed to extract symbol name from message "${message.text}"`, - ); - } - const [, symbolName] = symbolMatch; - - const sourceFile = - message.sourceFilePath && - program.getSourceFile(message.sourceFilePath); - if (!sourceFile) { - throw new Error( - `Failed to find source file in program at path "${message.sourceFilePath}"`, - ); - } - - // The local name of the symbol within the file, rather than the exported name - let localName = (sourceFile as any).identifiers?.get(symbolName); - - if (!localName) { - // Sometimes the symbol name is suffixed with a number to disambiguate, - // e.g. "Props_14" instead of "Props" if there are multiple Props interfaces - // so we try to strip that suffix and look up the symbol again. - const [, trimmedSymbolName] = symbolName.match(/(.*)_\d+/) || []; - localName = (sourceFile as any).identifiers?.get(trimmedSymbolName); - } - - if (!localName) { - throw new Error( - `Unable to find local name of "${symbolName}" in ${sourceFile.fileName}`, - ); - } - - // The local AST node of the export that we're missing - const local = (sourceFile as any).locals?.get(localName); - if (!local) { - return true; - } - - // Use the type checker to look up the actual declaration(s) rather than the one in the local file - const type = program.getTypeChecker().getDeclaredTypeOfSymbol(local); - if (!type) { - throw new Error( - `Unable to find type declaration of "${symbolName}" in ${sourceFile.fileName}`, - ); - } - const declarations = type.aliasSymbol?.declarations; - if (!declarations || declarations.length === 0) { - return true; - } - - // If any of the TSDoc comments contain a @ignore tag, we ignore this message - const isIgnored = declarations.some(declaration => { - const tags = [(declaration as any).jsDoc] - .flat() - .filter(Boolean) - .flatMap((tagNode: any) => tagNode.tags); - - return tags.some(tag => tag?.tagName.text === 'ignore'); - }); - - return !isIgnored; - }); - }, - ); - - const content = originalGenerateReviewFileContent.call( - this, - collector, - ...moreArgs, - ); - - try { - const prettier = require('prettier') as typeof import('prettier'); - - const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {}; - return prettier.format(content, { - ...config, - parser: 'markdown', - }); - } catch (e) { - return content; - } - }; - -export async function createTemporaryTsConfig(includedPackageDirs: string[]) { - const path = cliPaths.resolveTargetRoot('tsconfig.tmp.json'); - - process.once('exit', () => { - fs.removeSync(path); - }); - - let assetTypeFile: string[] = []; - - try { - assetTypeFile = [ - require.resolve('@backstage/cli/asset-types/asset-types.d.ts'), - ]; - } catch { - /** ignore */ - } - - await fs.writeJson(path, { - extends: './tsconfig.json', - include: [ - // These two contain global definitions that are needed for stable API report generation - ...assetTypeFile, - ...includedPackageDirs.map(dir => join(dir, 'src')), - ], - // we don't exclude node_modules so that we can use the asset-types.d.ts file - exclude: [], - }); - - return path; -} - -export async function countApiReportWarnings(reportPath: string) { - try { - const content = await fs.readFile(reportPath, 'utf8'); - const lines = content.split('\n'); - - const lineWarnings = lines.filter(line => - line.includes('// Warning:'), - ).length; - - const trailerStart = lines.findIndex( - line => line === '// Warnings were encountered during analysis:', - ); - const trailerWarnings = - trailerStart === -1 - ? 0 - : lines.length - - trailerStart - - 4; /* 4 lines at the trailer and after are not warnings */ - - return lineWarnings + trailerWarnings; - } catch (error) { - if (error.code === 'ENOENT') { - return 0; - } - throw error; - } -} - -export async function getTsDocConfig() { - const tsdocConfigFile = await TSDocConfigFile.loadFile( - require.resolve('@microsoft/api-extractor/extends/tsdoc-base.json'), - ); - tsdocConfigFile.addTagDefinition({ - tagName: '@ignore', - syntaxKind: TSDocTagSyntaxKind.ModifierTag, - }); - tsdocConfigFile.addTagDefinition({ - tagName: '@config', - syntaxKind: TSDocTagSyntaxKind.BlockTag, - }); - tsdocConfigFile.setSupportForTag('@ignore', true); - tsdocConfigFile.setSupportForTag('@config', true); - return tsdocConfigFile; -} export function logApiReportInstructions() { console.log(''); @@ -295,920 +34,6 @@ export function logApiReportInstructions() { console.log(''); } -async function findPackageEntryPoints(packageDirs: string[]): Promise< - Array<{ - // package dir relative to root, e.g. "packages/backend-app-api" - packageDir: string; - // the name of the export, e.g. "index" or "alpha" - name: string; - // the path within the dist directory for this export, e.g. "alpha.d.ts" - distPath: string; - // the path within the dist-types directory of this package for this export, - // e.g. "src/entrypoints/foo/index.d.ts" - distTypesPath: string; - }> -> { - return Promise.all( - packageDirs.map(async packageDir => { - const pkg = await fs.readJson( - cliPaths.resolveTargetRoot(packageDir, 'package.json'), - ); - - return getPackageExportDetails(pkg).map(details => { - return { packageDir, ...details }; - }); - }), - ).then(results => results.flat()); -} - -interface ApiExtractionOptions { - packageDirs: string[]; - outputDir: string; - isLocalBuild: boolean; - tsconfigFilePath: string; - allowWarnings?: boolean | string[]; - omitMessages?: string[]; - validateReleaseTags?: boolean; -} - -export async function runApiExtraction({ - packageDirs, - outputDir, - isLocalBuild, - tsconfigFilePath, - allowWarnings = false, - omitMessages = [], - validateReleaseTags = false, -}: ApiExtractionOptions) { - await fs.remove(outputDir); - - // The collection of all entry points of all packages, as a single list - const allEntryPoints = await findPackageEntryPoints(packageDirs); - - // The path (relative to the root) to ALL dist-types entry points (e.g. - // "dist-types/packages/backend-app-api/src/index.d.ts"). These are used as - // "extra"/contextual entry points for the extractor so that it can see the - // full context of things that are required by the local entry point being - // inspected. - const allDistTypesEntryPointPaths = allEntryPoints.map( - ({ packageDir, distTypesPath }) => { - return cliPaths.resolveTargetRoot( - './dist-types', - packageDir, - distTypesPath, - ); - }, - ); - - let compilerState: CompilerState | undefined = undefined; - - const allowWarningPkg = Array.isArray(allowWarnings) ? allowWarnings : []; - - const messagesConf: { [key: string]: { logLevel: string } } = {}; - for (const messageCode of omitMessages) { - messagesConf[messageCode] = { - logLevel: 'none', - }; - } - - const warnings = new Array(); - - for (const [packageDir, packageEntryPoints] of Object.entries( - groupBy(allEntryPoints, ep => ep.packageDir), - )) { - console.log(`## Processing ${packageDir}`); - const noBail = Array.isArray(allowWarnings) - ? allowWarnings.some(aw => aw === packageDir || minimatch(packageDir, aw)) - : allowWarnings; - - const projectFolder = cliPaths.resolveTargetRoot(packageDir); - const packageFolder = cliPaths.resolveTargetRoot( - './dist-types', - packageDir, - ); - - const remainingReportFiles = new Set( - fs.readdirSync(projectFolder).filter( - filename => - // https://regex101.com/r/QDZIV0/2 - filename !== 'knip-report.md' && - !filename.endsWith('.sql.md') && - // this has to temporarily match all old api report formats - filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/), - ), - ); - - for (const packageEntryPoint of packageEntryPoints) { - const suffix = - packageEntryPoint.name === 'index' ? '' : `-${packageEntryPoint.name}`; - const reportFileName = `report${suffix}`; - const reportPath = resolvePath(projectFolder, `${reportFileName}.api.md`); - - const warningCountBefore = await countApiReportWarnings(reportPath); - - const extractorConfig = ExtractorConfig.prepare({ - configObject: { - mainEntryPointFilePath: resolvePath( - packageFolder, - packageEntryPoint.distTypesPath, - ), - bundledPackages: [], - - compiler: { - tsconfigFilePath, - }, - - apiReport: { - enabled: true, - reportFileName, - reportFolder: projectFolder, - reportTempFolder: resolvePath( - outputDir, - `${suffix}`, - ), - }, - - docModel: { - // TODO(Rugvip): This skips docs for non-index entry points. We can try to work around it, but - // most likely it makes sense to wait for API Extractor to natively support exports. - enabled: packageEntryPoint.name === 'index', - apiJsonFilePath: resolvePath( - outputDir, - `${suffix}.api.json`, - ), - }, - - dtsRollup: { - enabled: false, - }, - - tsdocMetadata: { - enabled: false, - }, - - messages: { - // Silence compiler warnings, as these will prevent the CI build to work - compilerMessageReporting: { - default: { - logLevel: 'none' as ExtractorLogLevel.None, - // These contain absolute file paths, so can't be included in the report - // addToApiReportFile: true, - }, - }, - extractorMessageReporting: { - default: { - logLevel: 'warning' as ExtractorLogLevel.Warning, - addToApiReportFile: true, - }, - ...messagesConf, - }, - tsdocMessageReporting: { - default: { - logLevel: 'warning' as ExtractorLogLevel.Warning, - addToApiReportFile: true, - }, - }, - }, - - newlineKind: 'lf', - - projectFolder, - }, - configObjectFullPath: projectFolder, - packageJsonFullPath: resolvePath(projectFolder, 'package.json'), - tsdocConfigFile: await getTsDocConfig(), - ignoreMissingEntryPoint: true, - }); - - // remove extracted reports from current list - for (const reportConfig of extractorConfig.reportConfigs) { - remainingReportFiles.delete(reportConfig.fileName); - } - - // The `packageFolder` needs to point to the location within `dist-types` in order for relative - // paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`, - // which needs to point to the actual file, so we override `packageFolder` afterwards. - ( - extractorConfig as { - packageFolder: string; - } - ).packageFolder = packageFolder; - - if (!compilerState) { - compilerState = CompilerState.create(extractorConfig, { - additionalEntryPoints: allDistTypesEntryPointPaths, - }); - } - - // Message verbosity can't be configured, so just skip the check instead - (Extractor as any)._checkCompilerCompatibility = () => {}; - - let shouldLogInstructions = false; - let conflictingFile: undefined | string = undefined; - - // Invoke API Extractor - const extractorResult = Extractor.invoke(extractorConfig, { - localBuild: isLocalBuild, - showVerboseMessages: false, - showDiagnostics: false, - messageCallback(message) { - if (message.text.includes('The API report file is missing')) { - shouldLogInstructions = true; - } - - // Detect messages like the following being output by the generator: - // Warning: You have changed the API signature for this project. Please copy the file "/home/runner/work/backstage/backstage/node_modules/.cache/api-extractor/backend-test-utils/report.api.md" to "report.api.md", or perform a local build (which does this automatically). See the Git repo documentation for more info. - if ( - message.text.includes( - 'You have changed the API signature for this project.', - ) - ) { - shouldLogInstructions = true; - const match = message.text.match( - /Please copy the file "(.*)" to "report\.api\.md"/, - ); - if (match) { - conflictingFile = match[1]; - } - } - }, - compilerState, - }); - - // This release tag validation makes sure that the release tag of known entry points match expectations. - // The root index entry point is only allowed @public exports, while /alpha and /beta only allow @alpha and @beta. - if ( - validateReleaseTags && - fs.pathExistsSync(extractorConfig.reportFilePath) - ) { - if (['index', 'alpha', 'beta'].includes(packageEntryPoint.name)) { - const report = await fs.readFile( - extractorConfig.reportFilePath, - 'utf8', - ); - const lines = report.split(/\r?\n/); - const expectedTag = - packageEntryPoint.name === 'index' - ? 'public' - : packageEntryPoint.name; - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i]; - const match = line.match(/^\/\/ @(alpha|beta|public)/); - if (match && match[1] !== expectedTag) { - // Because of limitations in the type script rollup logic we need to allow public exports from the other release stages - // TODO(Rugvip): Try to work around the need for this exception - if (expectedTag !== 'public' && match[1] === 'public') { - continue; - } - throw new Error( - `Unexpected release tag ${match[1]} in ${ - extractorConfig.reportFilePath - } at line ${i + 1}`, - ); - } - } - } - } - - if (!extractorResult.succeeded) { - if (shouldLogInstructions) { - logApiReportInstructions(); - - if (conflictingFile) { - console.log(''); - console.log( - `The conflicting file is ${relativePath( - tmpDir, - conflictingFile, - )}, with the following content:`, - ); - console.log(''); - - const content = await fs.readFile(conflictingFile, 'utf8'); - console.log(content); - - logApiReportInstructions(); - } - } - - throw new Error( - `API Extractor completed with ${extractorResult.errorCount} errors` + - ` and ${extractorResult.warningCount} warnings`, - ); - } - - const warningCountAfter = await countApiReportWarnings(reportPath); - - if (noBail) { - console.log(`Skipping warnings check for ${packageDir}`); - } - if (warningCountAfter > 0 && !noBail) { - throw new Error( - `The API Report for ${packageDir} is not allowed to have warnings`, - ); - } - if (warningCountAfter === 0 && allowWarningPkg.includes(packageDir)) { - console.log( - `No need to allow warnings for ${packageDir}, it does not have any`, - ); - } - if (warningCountAfter > warningCountBefore) { - warnings.push( - `The API Report for ${packageDir} introduces new warnings. ` + - 'Please fix these warnings in order to keep the API Reports tidy.', - ); - } - } - - if (remainingReportFiles.size > 0) { - if (isLocalBuild) { - for (const f of remainingReportFiles) { - fs.rmSync(resolvePath(projectFolder, f)); - console.log(`Deleted deprecated API report ${f}`); - } - } else { - const staleList = [...remainingReportFiles] - .map(f => join(packageDir, f)) - .join(', '); - throw new Error( - `The API Report(s) ${staleList} are no longer relevant and should be deleted`, - ); - } - } - } - - if (warnings.length > 0) { - console.warn(); - for (const warning of warnings) { - console.warn(warning); - } - console.warn(); - } -} - -/* -WARNING: Bring a blanket if you're gonna read the code below - -There's some weird shit going on here, and it's because we cba -forking rushstack to modify the api-documenter markdown generation, -which otherwise is the recommended way to do customizations. -*/ - -type ExcerptToken = { - kind: string; - text: string; - canonicalReference?: string; -}; - -class ExcerptTokenMatcher { - readonly #tokens: ExcerptToken[]; - - constructor(tokens: ExcerptToken[]) { - this.#tokens = tokens.slice(); - } - - nextContent() { - const token = this.#tokens.shift(); - if (token?.kind === 'Content') { - return token.text; - } - return undefined; - } - - matchContent(expectedText: string) { - const text = this.nextContent(); - return text !== expectedText; - } - - getTokensUntilArrow() { - const tokens = []; - for (;;) { - const token = this.#tokens.shift(); - if (token === undefined) { - return undefined; - } - if (token.kind === 'Content' && token.text === ') => ') { - return tokens; - } - tokens.push(token); - } - } - - getComponentReturnTokens() { - const first = this.#tokens.shift(); - if (!first) { - return undefined; - } - const second = this.#tokens.shift(); - - if (this.#tokens.length !== 0) { - return undefined; - } - if (first.kind !== 'Reference' || first.text !== 'JSX.Element') { - return undefined; - } - if (!second) { - return [first]; - } else if (second.kind === 'Content' && second.text === ' | null') { - return [first, second]; - } - return undefined; - } -} - -class ApiModelTransforms { - static deserializeWithTransforms( - serialized: any, - transforms: Array<(member: any) => any>, - ): ApiPackage { - if (serialized.kind !== 'Package') { - throw new Error( - `Unexpected root kind in serialized ApiPackage, ${serialized.kind}`, - ); - } - if (serialized.members.length !== 1) { - throw new Error( - `Unexpected members in serialized ApiPackage, [${serialized.members - .map((m: { kind: any }) => m.kind) - .join(' ')}]`, - ); - } - const [entryPoint] = serialized.members; - if (entryPoint.kind !== 'EntryPoint') { - throw new Error( - `Unexpected kind in serialized ApiPackage member, ${entryPoint.kind}`, - ); - } - - const transformed = { - ...serialized, - members: [ - { - ...entryPoint, - members: entryPoint.members.map((member: any) => - transforms.reduce((m, t) => t(m), member), - ), - }, - ], - }; - - return ApiPackage.deserialize( - transformed, - transformed.metadata, - ) as ApiPackage; - } - - static transformArrowComponents = (member: any) => { - if (member.kind !== 'Variable') { - return member; - } - - const { name, excerptTokens } = member; - - // First letter in name must be uppercase - const [firstChar] = name; - if (firstChar.toLocaleUpperCase('en-US') !== firstChar) { - return member; - } - - // First content must match expected declaration format - const tokens = new ExcerptTokenMatcher(excerptTokens); - if (tokens.nextContent() !== `${name}: `) { - return member; - } - - // Next needs to be an arrow with `props` parameters or no parameters - // followed by a return type of `JSX.Element | null` or just `JSX.Element` - const declStart = tokens.nextContent(); - if (declStart === '(props: ' || declStart === '(_props: ') { - const props = tokens.getTokensUntilArrow(); - const ret = tokens.getComponentReturnTokens(); - if (props && ret) { - return this.makeComponentMember(member, ret, props); - } - } else if (declStart === '() => ') { - const ret = tokens.getComponentReturnTokens(); - if (ret) { - return this.makeComponentMember(member, ret); - } - } - return member; - }; - - static makeComponentMember( - member: any, - ret: ExcerptToken[], - props?: ExcerptToken[], - ) { - const declTokens = props - ? [ - { - kind: 'Content', - text: `export declare function ${member.name}(props: `, - }, - ...props, - { - kind: 'Content', - text: '): ', - }, - ] - : [ - { - kind: 'Content', - text: `export declare function ${member.name}(): `, - }, - ]; - - return { - kind: 'Function', - name: member.name, - releaseTag: member.releaseTag, - docComment: member.docComment ?? '', - canonicalReference: member.canonicalReference, - excerptTokens: [...declTokens, ...ret], - returnTypeTokenRange: { - startIndex: declTokens.length, - endIndex: declTokens.length + ret.length, - }, - parameters: props - ? [ - { - parameterName: 'props', - parameterTypeTokenRange: { - startIndex: 1, - endIndex: 1 + props.length, - }, - }, - ] - : [], - overloadIndex: 1, - }; - } - - static transformTrimDeclare = (member: any) => { - const { excerptTokens } = member; - const firstContent = new ExcerptTokenMatcher(excerptTokens).nextContent(); - if (firstContent && firstContent.startsWith('export declare ')) { - return { - ...member, - excerptTokens: [ - { - kind: 'Content', - text: firstContent.slice('export declare '.length), - }, - ...excerptTokens.slice(1), - ], - }; - } - return member; - }; -} - -export async function buildDocs({ - inputDir, - outputDir, -}: { - inputDir: string; - outputDir: string; -}) { - // We start by constructing our own model from the files so that - // we get a change to modify them, as the model is otherwise read-only. - const parseFile = async (filename: string): Promise => { - console.log(`Reading ${filename}`); - return fs.readJson(resolvePath(inputDir, filename)); - }; - - const filenames = await fs.readdir(inputDir); - const serializedPackages = await Promise.all( - filenames - .filter(filename => filename.match(/\.api\.json$/i)) - .map(parseFile), - ); - - const newModel = new ApiModel(); - for (const serialized of serializedPackages) { - newModel.addMember( - ApiModelTransforms.deserializeWithTransforms(serialized, [ - ApiModelTransforms.transformArrowComponents, - ApiModelTransforms.transformTrimDeclare, - ]), - ); - } - - // The doc AST need to be extended with custom nodes if we want to - // add any extra content. - // This one is for the YAML front matter that we need for the microsite. - class DocFrontMatter extends DocNode { - static kind = 'DocFrontMatter'; - - public readonly values: { [name: string]: unknown }; - - public constructor( - parameters: IDocNodeContainerParameters & { - values: { [name: string]: unknown }; - }, - ) { - super(parameters); - this.values = parameters.values; - } - - /** @override */ - public get kind(): string { - return DocFrontMatter.kind; - } - } - - // This class only propose is to have a different kind and be able to render links with backticks - class DocCodeSpanLink extends DocLinkTag { - static kind = 'DocCodeSpanLink'; - - /** @override */ - public get kind(): string { - return DocCodeSpanLink.kind; - } - } - - // This is where we actually write the markdown and where we can hook - // in the rendering of our own nodes. - class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter { - // Until https://github.com/microsoft/rushstack/issues/2914 gets fixed or we change markdown renderer we need a fix - // to render pipe | character correctly. - protected getEscapedText(text: string): string { - return text - .replace(/\\/g, '\\\\') // first replace the escape character - .replace(/[*#[\]_`~]/g, x => `\\${x}`) // then escape any special characters - .replace(/---/g, '\\-\\-\\-') // hyphens only if it's 3 or more - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/\|/g, '|'); - } - /** @override */ - protected writeNode( - docNode: DocNode, - context: IMarkdownEmitterContext, - docNodeSiblings: boolean, - ): void { - switch (docNode.kind) { - case DocFrontMatter.kind: { - const node = docNode as DocFrontMatter; - context.writer.writeLine('---'); - for (const [name, value] of Object.entries(node.values)) { - if (value) { - context.writer.writeLine( - `${name}: "${String(value).replace(/\"/g, '')}"`, - ); - } - } - context.writer.writeLine('---'); - context.writer.writeLine(); - break; - } - case 'BlockTag': { - const node = docNode as DocBlockTag; - if (node.tagName === '@config') { - context.writer.writeLine('## Related config '); - } - break; - } - case DocCodeSpanLink.kind: { - const node = docNode as DocLinkTag; - if (node.codeDestination) { - // TODO @sarabadu understand if we need `codeDestination` at all on this custom DocCodeSpanLink - super.writeLinkTagWithCodeDestination(node, context); - } else if (node.urlDestination) { - const linkText = - node.linkText !== undefined ? node.linkText : node.urlDestination; - const encodedLinkText = this.getEscapedText( - linkText.replace(/\s+/g, ' '), - ); - context.writer.write('['); - context.writer.write(`\`${encodedLinkText}\``); - context.writer.write(`](${node.urlDestination})`); - } else if (node.linkText) { - this.writePlainText(node.linkText, context); - } - break; - } - default: - super.writeNode(docNode, context, docNodeSiblings); - } - } - - /** @override */ - emit( - stringBuilder: any, - docNode: DocNode, - options: ICustomMarkdownEmitterOptions, - ) { - // Hack to get rid of the leading comment of each file, since - // we want the front matter to come first - stringBuilder._chunks.length = 0; - return super.emit(stringBuilder, docNode, options); - } - } - - class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) { - constructor(options: IMarkdownDocumenterOptions) { - super(options); - - // It's a strict model, we gotta register the allowed usage of our new node - ( - this._tsdocConfiguration as TSDocConfiguration - ).docNodeManager.registerDocNodes('@backstage/docs', [ - { docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter }, - ]); - ( - this._tsdocConfiguration as TSDocConfiguration - ).docNodeManager.registerDocNodes('@backstage/docs', [ - { docNodeKind: DocCodeSpanLink.kind, constructor: DocCodeSpanLink }, - ]); - ( - this._tsdocConfiguration as TSDocConfiguration - ).docNodeManager.registerAllowableChildren('Paragraph', [ - DocFrontMatter.kind, - DocCodeSpanLink.kind, - ]); - - const def = { - tagName: '@config', - syntaxKind: TSDocTagSyntaxKind.BlockTag, - tagNameWithUpperCase: '@CONFIG', - standardization: Standardization.Extended, - allowMultiple: false, - }; - (this._tsdocConfiguration as TSDocConfiguration).addTagDefinition(def); - (this._tsdocConfiguration as TSDocConfiguration).setSupportForTag( - def, - true, - ); - this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel); - } - - private _getFilenameForApiItem(apiItem: ApiItem): string { - const filename: string = super._getFilenameForApiItem(apiItem); - - if (filename.includes('.html.')) { - return filename.replace(/\.html\./g, '._html.'); - } - - return filename; - } - - // We don't really get many chances to modify the generated AST - // so we hook in wherever we can. In this case we add the front matter - // just before writing the breadcrumbs at the top. - /** @override */ - _writeBreadcrumb(output: any, apiItem: ApiItem & { name: string }) { - let title; - let description; - - const name = apiItem.getScopedNameWithinPackage(); - if (name) { - title = name; - description = `API reference for ${apiItem.getScopedNameWithinPackage()}`; - } else if (apiItem.kind === 'Model') { - title = 'Package Index'; - description = 'Index of all Backstage Packages'; - } else if (apiItem.name) { - title = apiItem.name; - description = `API Reference for ${apiItem.name}`; - } else { - title = apiItem.displayName; - description = `API Reference for ${apiItem.displayName}`; - } - - // Add our front matter - output.appendNodeInParagraph( - new DocFrontMatter({ - configuration: this._tsdocConfiguration, - values: { - id: this._getFilenameForApiItem(apiItem).slice(0, -3), - title, - description, - }, - }), - ); - - const configuration: TSDocConfiguration = this._tsdocConfiguration; - - output.appendNodeInParagraph( - new DocLinkTag({ - configuration, - tagName: '@link', - linkText: 'Home', - urlDestination: this._getLinkFilenameForApiItem(this._apiModel), - }), - ); - - for (const hierarchyItem of apiItem.getHierarchy()) { - switch (hierarchyItem.kind) { - case ApiItemKind.Model: - case ApiItemKind.EntryPoint: - // We don't show the model as part of the breadcrumb because it is the root-level container. - // We don't show the entry point because today API Extractor doesn't support multiple entry points; - // this may change in the future. - break; - default: - output.appendNodesInParagraph([ - new DocPlainText({ - configuration, - text: ' > ', - }), - new DocCodeSpanLink({ - configuration, - tagName: '@link', - linkText: hierarchyItem.displayName, - urlDestination: this._getLinkFilenameForApiItem(hierarchyItem), - }), - ]); - } - } - - // We wanna ignore the header that always gets written after the breadcrumb - // This otherwise becomes more or less a duplicate of the title in the front matter - const oldAppendNode = output.appendNode; - output.appendNode = () => { - output.appendNode = oldAppendNode; - }; - } - - _writeModelTable( - output: { appendNode: (arg0: DocTable | DocHeading) => void }, - apiModel: { members: any }, - ): void { - const configuration = this._tsdocConfiguration; - - const packagesTable = new DocTable({ - configuration, - headerTitles: ['Package', 'Description'], - }); - - const pluginsTable = new DocTable({ - configuration, - headerTitles: ['Package', 'Description'], - }); - - for (const apiMember of apiModel.members) { - const row = new DocTableRow({ configuration }, [ - this._createTitleCell(apiMember), - this._createDescriptionCell(apiMember), - ]); - - if (apiMember.kind === 'Package') { - this._writeApiItemPage(apiMember); - - if (apiMember.name.startsWith('@backstage/plugin-')) { - pluginsTable.addRow(row); - } else { - packagesTable.addRow(row); - } - } - } - - if (packagesTable.rows.length > 0) { - output.appendNode( - new DocHeading({ - configuration: this._tsdocConfiguration, - title: 'Packages', - }), - ); - output.appendNode(packagesTable); - } - - if (pluginsTable.rows.length > 0) { - output.appendNode( - new DocHeading({ - configuration: this._tsdocConfiguration, - title: 'Plugins', - }), - ); - output.appendNode(pluginsTable); - } - } - } - - // This is root of the documentation generation, but it's not directly - // responsible for generating markdown, it just constructs an AST that - // is the consumed by an emitter to actually write the files. - const documenter = new CustomMarkdownDocumenter({ - apiModel: newModel, - documenterConfig: { - outputTarget: 'markdown', - newlineKind: '\n', - // De ba dålig kod - configFilePath: '', - configFile: {}, - } as any, - outputFolder: outputDir, - }); - - // Clean up existing stuff and write ALL the docs! - await fs.remove(outputDir); - await fs.ensureDir(outputDir); - documenter.generateFiles(); -} - export async function categorizePackageDirs(packageDirs: string[]) { const dirs = packageDirs.slice(); const tsPackageDirs = new Array(); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts index c78e7179b3..796b733d97 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -18,16 +18,13 @@ import { createMockDirectory } from '@backstage/backend-test-utils'; import { normalize } from 'path'; import * as pathsLib from '../../lib/paths'; -import { - buildDocs, - categorizePackageDirs, - runApiExtraction, -} from './api-extractor'; +import { categorizePackageDirs } from './api-extractor'; import { buildApiReports } from './api-reports'; import { generateTypeDeclarations } from './generateTypeDeclarations'; import { PackageGraph } from '@backstage/cli-node'; import { runCliExtraction } from './cli-reports'; +import { runApiExtraction, buildDocs } from './api-reports/index'; jest.mock('./generateTypeDeclarations'); // create mocks for the dependencies of the `buildApiReports` function diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 93780bf86d..d02a668af6 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -15,16 +15,16 @@ */ import { OptionValues } from 'commander'; -import { - buildDocs, - categorizePackageDirs, - createTemporaryTsConfig, - runApiExtraction, -} from './api-extractor'; +import { categorizePackageDirs } from './api-extractor'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; import { runSqlExtraction } from './sql-reports'; import { runCliExtraction } from './cli-reports'; +import { + runApiExtraction, + buildDocs, + createTemporaryTsConfig, +} from './api-reports/index'; type Options = { ci?: boolean; diff --git a/packages/repo-tools/src/commands/api-reports/api-reports/buildDocs.ts b/packages/repo-tools/src/commands/api-reports/api-reports/buildDocs.ts new file mode 100644 index 0000000000..42b585818d --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports/buildDocs.ts @@ -0,0 +1,609 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import fs from 'fs-extra'; +import { + DocBlockTag, + DocLinkTag, + DocNode, + DocPlainText, + IDocNodeContainerParameters, + Standardization, + TSDocConfiguration, + TSDocTagSyntaxKind, +} from '@microsoft/tsdoc'; +import { + ApiItem, + ApiItemKind, + ApiModel, + ApiPackage, +} from '@microsoft/api-extractor-model'; +import { + IMarkdownDocumenterOptions, + MarkdownDocumenter, +} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter'; +import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable'; +import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow'; +import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading'; +import { + CustomMarkdownEmitter, + ICustomMarkdownEmitterOptions, +} from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; +import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; + +/* +WARNING: Bring a blanket if you're gonna read the code below + +There's some weird shit going on here, and it's because we cba +forking rushstack to modify the api-documenter markdown generation, +which otherwise is the recommended way to do customizations. +*/ + +type ExcerptToken = { + kind: string; + text: string; + canonicalReference?: string; +}; + +class ExcerptTokenMatcher { + readonly #tokens: ExcerptToken[]; + + constructor(tokens: ExcerptToken[]) { + this.#tokens = tokens.slice(); + } + + nextContent() { + const token = this.#tokens.shift(); + if (token?.kind === 'Content') { + return token.text; + } + return undefined; + } + + matchContent(expectedText: string) { + const text = this.nextContent(); + return text !== expectedText; + } + + getTokensUntilArrow() { + const tokens = []; + for (;;) { + const token = this.#tokens.shift(); + if (token === undefined) { + return undefined; + } + if (token.kind === 'Content' && token.text === ') => ') { + return tokens; + } + tokens.push(token); + } + } + + getComponentReturnTokens() { + const first = this.#tokens.shift(); + if (!first) { + return undefined; + } + const second = this.#tokens.shift(); + + if (this.#tokens.length !== 0) { + return undefined; + } + if (first.kind !== 'Reference' || first.text !== 'JSX.Element') { + return undefined; + } + if (!second) { + return [first]; + } else if (second.kind === 'Content' && second.text === ' | null') { + return [first, second]; + } + return undefined; + } +} + +class ApiModelTransforms { + static deserializeWithTransforms( + serialized: any, + transforms: Array<(member: any) => any>, + ): ApiPackage { + if (serialized.kind !== 'Package') { + throw new Error( + `Unexpected root kind in serialized ApiPackage, ${serialized.kind}`, + ); + } + if (serialized.members.length !== 1) { + throw new Error( + `Unexpected members in serialized ApiPackage, [${serialized.members + .map((m: { kind: any }) => m.kind) + .join(' ')}]`, + ); + } + const [entryPoint] = serialized.members; + if (entryPoint.kind !== 'EntryPoint') { + throw new Error( + `Unexpected kind in serialized ApiPackage member, ${entryPoint.kind}`, + ); + } + + const transformed = { + ...serialized, + members: [ + { + ...entryPoint, + members: entryPoint.members.map((member: any) => + transforms.reduce((m, t) => t(m), member), + ), + }, + ], + }; + + return ApiPackage.deserialize( + transformed, + transformed.metadata, + ) as ApiPackage; + } + + static transformArrowComponents = (member: any) => { + if (member.kind !== 'Variable') { + return member; + } + + const { name, excerptTokens } = member; + + // First letter in name must be uppercase + const [firstChar] = name; + if (firstChar.toLocaleUpperCase('en-US') !== firstChar) { + return member; + } + + // First content must match expected declaration format + const tokens = new ExcerptTokenMatcher(excerptTokens); + if (tokens.nextContent() !== `${name}: `) { + return member; + } + + // Next needs to be an arrow with `props` parameters or no parameters + // followed by a return type of `JSX.Element | null` or just `JSX.Element` + const declStart = tokens.nextContent(); + if (declStart === '(props: ' || declStart === '(_props: ') { + const props = tokens.getTokensUntilArrow(); + const ret = tokens.getComponentReturnTokens(); + if (props && ret) { + return this.makeComponentMember(member, ret, props); + } + } else if (declStart === '() => ') { + const ret = tokens.getComponentReturnTokens(); + if (ret) { + return this.makeComponentMember(member, ret); + } + } + return member; + }; + + static makeComponentMember( + member: any, + ret: ExcerptToken[], + props?: ExcerptToken[], + ) { + const declTokens = props + ? [ + { + kind: 'Content', + text: `export declare function ${member.name}(props: `, + }, + ...props, + { + kind: 'Content', + text: '): ', + }, + ] + : [ + { + kind: 'Content', + text: `export declare function ${member.name}(): `, + }, + ]; + + return { + kind: 'Function', + name: member.name, + releaseTag: member.releaseTag, + docComment: member.docComment ?? '', + canonicalReference: member.canonicalReference, + excerptTokens: [...declTokens, ...ret], + returnTypeTokenRange: { + startIndex: declTokens.length, + endIndex: declTokens.length + ret.length, + }, + parameters: props + ? [ + { + parameterName: 'props', + parameterTypeTokenRange: { + startIndex: 1, + endIndex: 1 + props.length, + }, + }, + ] + : [], + overloadIndex: 1, + }; + } + + static transformTrimDeclare = (member: any) => { + const { excerptTokens } = member; + const firstContent = new ExcerptTokenMatcher(excerptTokens).nextContent(); + if (firstContent && firstContent.startsWith('export declare ')) { + return { + ...member, + excerptTokens: [ + { + kind: 'Content', + text: firstContent.slice('export declare '.length), + }, + ...excerptTokens.slice(1), + ], + }; + } + return member; + }; +} + +export async function buildDocs({ + inputDir, + outputDir, +}: { + inputDir: string; + outputDir: string; +}) { + // We start by constructing our own model from the files so that + // we get a change to modify them, as the model is otherwise read-only. + const parseFile = async (filename: string): Promise => { + console.log(`Reading ${filename}`); + return fs.readJson(resolvePath(inputDir, filename)); + }; + + const filenames = await fs.readdir(inputDir); + const serializedPackages = await Promise.all( + filenames + .filter(filename => filename.match(/\.api\.json$/i)) + .map(parseFile), + ); + + const newModel = new ApiModel(); + for (const serialized of serializedPackages) { + newModel.addMember( + ApiModelTransforms.deserializeWithTransforms(serialized, [ + ApiModelTransforms.transformArrowComponents, + ApiModelTransforms.transformTrimDeclare, + ]), + ); + } + + // The doc AST need to be extended with custom nodes if we want to + // add any extra content. + // This one is for the YAML front matter that we need for the microsite. + class DocFrontMatter extends DocNode { + static kind = 'DocFrontMatter'; + + public readonly values: { [name: string]: unknown }; + + public constructor( + parameters: IDocNodeContainerParameters & { + values: { [name: string]: unknown }; + }, + ) { + super(parameters); + this.values = parameters.values; + } + + /** @override */ + public get kind(): string { + return DocFrontMatter.kind; + } + } + + // This class only propose is to have a different kind and be able to render links with backticks + class DocCodeSpanLink extends DocLinkTag { + static kind = 'DocCodeSpanLink'; + + /** @override */ + public get kind(): string { + return DocCodeSpanLink.kind; + } + } + + // This is where we actually write the markdown and where we can hook + // in the rendering of our own nodes. + class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter { + // Until https://github.com/microsoft/rushstack/issues/2914 gets fixed or we change markdown renderer we need a fix + // to render pipe | character correctly. + protected getEscapedText(text: string): string { + return text + .replace(/\\/g, '\\\\') // first replace the escape character + .replace(/[*#[\]_`~]/g, x => `\\${x}`) // then escape any special characters + .replace(/---/g, '\\-\\-\\-') // hyphens only if it's 3 or more + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/\|/g, '|'); + } + /** @override */ + protected writeNode( + docNode: DocNode, + context: IMarkdownEmitterContext, + docNodeSiblings: boolean, + ): void { + switch (docNode.kind) { + case DocFrontMatter.kind: { + const node = docNode as DocFrontMatter; + context.writer.writeLine('---'); + for (const [name, value] of Object.entries(node.values)) { + if (value) { + context.writer.writeLine( + `${name}: "${String(value).replace(/\"/g, '')}"`, + ); + } + } + context.writer.writeLine('---'); + context.writer.writeLine(); + break; + } + case 'BlockTag': { + const node = docNode as DocBlockTag; + if (node.tagName === '@config') { + context.writer.writeLine('## Related config '); + } + break; + } + case DocCodeSpanLink.kind: { + const node = docNode as DocLinkTag; + if (node.codeDestination) { + // TODO @sarabadu understand if we need `codeDestination` at all on this custom DocCodeSpanLink + super.writeLinkTagWithCodeDestination(node, context); + } else if (node.urlDestination) { + const linkText = + node.linkText !== undefined ? node.linkText : node.urlDestination; + const encodedLinkText = this.getEscapedText( + linkText.replace(/\s+/g, ' '), + ); + context.writer.write('['); + context.writer.write(`\`${encodedLinkText}\``); + context.writer.write(`](${node.urlDestination})`); + } else if (node.linkText) { + this.writePlainText(node.linkText, context); + } + break; + } + default: + super.writeNode(docNode, context, docNodeSiblings); + } + } + + /** @override */ + emit( + stringBuilder: any, + docNode: DocNode, + options: ICustomMarkdownEmitterOptions, + ) { + // Hack to get rid of the leading comment of each file, since + // we want the front matter to come first + stringBuilder._chunks.length = 0; + return super.emit(stringBuilder, docNode, options); + } + } + + class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) { + constructor(options: IMarkdownDocumenterOptions) { + super(options); + + // It's a strict model, we gotta register the allowed usage of our new node + ( + this._tsdocConfiguration as TSDocConfiguration + ).docNodeManager.registerDocNodes('@backstage/docs', [ + { docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter }, + ]); + ( + this._tsdocConfiguration as TSDocConfiguration + ).docNodeManager.registerDocNodes('@backstage/docs', [ + { docNodeKind: DocCodeSpanLink.kind, constructor: DocCodeSpanLink }, + ]); + ( + this._tsdocConfiguration as TSDocConfiguration + ).docNodeManager.registerAllowableChildren('Paragraph', [ + DocFrontMatter.kind, + DocCodeSpanLink.kind, + ]); + + const def = { + tagName: '@config', + syntaxKind: TSDocTagSyntaxKind.BlockTag, + tagNameWithUpperCase: '@CONFIG', + standardization: Standardization.Extended, + allowMultiple: false, + }; + (this._tsdocConfiguration as TSDocConfiguration).addTagDefinition(def); + (this._tsdocConfiguration as TSDocConfiguration).setSupportForTag( + def, + true, + ); + this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel); + } + + private _getFilenameForApiItem(apiItem: ApiItem): string { + const filename: string = super._getFilenameForApiItem(apiItem); + + if (filename.includes('.html.')) { + return filename.replace(/\.html\./g, '._html.'); + } + + return filename; + } + + // We don't really get many chances to modify the generated AST + // so we hook in wherever we can. In this case we add the front matter + // just before writing the breadcrumbs at the top. + /** @override */ + _writeBreadcrumb(output: any, apiItem: ApiItem & { name: string }) { + let title; + let description; + + const name = apiItem.getScopedNameWithinPackage(); + if (name) { + title = name; + description = `API reference for ${apiItem.getScopedNameWithinPackage()}`; + } else if (apiItem.kind === 'Model') { + title = 'Package Index'; + description = 'Index of all Backstage Packages'; + } else if (apiItem.name) { + title = apiItem.name; + description = `API Reference for ${apiItem.name}`; + } else { + title = apiItem.displayName; + description = `API Reference for ${apiItem.displayName}`; + } + + // Add our front matter + output.appendNodeInParagraph( + new DocFrontMatter({ + configuration: this._tsdocConfiguration, + values: { + id: this._getFilenameForApiItem(apiItem).slice(0, -3), + title, + description, + }, + }), + ); + + const configuration: TSDocConfiguration = this._tsdocConfiguration; + + output.appendNodeInParagraph( + new DocLinkTag({ + configuration, + tagName: '@link', + linkText: 'Home', + urlDestination: this._getLinkFilenameForApiItem(this._apiModel), + }), + ); + + for (const hierarchyItem of apiItem.getHierarchy()) { + switch (hierarchyItem.kind) { + case ApiItemKind.Model: + case ApiItemKind.EntryPoint: + // We don't show the model as part of the breadcrumb because it is the root-level container. + // We don't show the entry point because today API Extractor doesn't support multiple entry points; + // this may change in the future. + break; + default: + output.appendNodesInParagraph([ + new DocPlainText({ + configuration, + text: ' > ', + }), + new DocCodeSpanLink({ + configuration, + tagName: '@link', + linkText: hierarchyItem.displayName, + urlDestination: this._getLinkFilenameForApiItem(hierarchyItem), + }), + ]); + } + } + + // We wanna ignore the header that always gets written after the breadcrumb + // This otherwise becomes more or less a duplicate of the title in the front matter + const oldAppendNode = output.appendNode; + output.appendNode = () => { + output.appendNode = oldAppendNode; + }; + } + + _writeModelTable( + output: { appendNode: (arg0: DocTable | DocHeading) => void }, + apiModel: { members: any }, + ): void { + const configuration = this._tsdocConfiguration; + + const packagesTable = new DocTable({ + configuration, + headerTitles: ['Package', 'Description'], + }); + + const pluginsTable = new DocTable({ + configuration, + headerTitles: ['Package', 'Description'], + }); + + for (const apiMember of apiModel.members) { + const row = new DocTableRow({ configuration }, [ + this._createTitleCell(apiMember), + this._createDescriptionCell(apiMember), + ]); + + if (apiMember.kind === 'Package') { + this._writeApiItemPage(apiMember); + + if (apiMember.name.startsWith('@backstage/plugin-')) { + pluginsTable.addRow(row); + } else { + packagesTable.addRow(row); + } + } + } + + if (packagesTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Packages', + }), + ); + output.appendNode(packagesTable); + } + + if (pluginsTable.rows.length > 0) { + output.appendNode( + new DocHeading({ + configuration: this._tsdocConfiguration, + title: 'Plugins', + }), + ); + output.appendNode(pluginsTable); + } + } + } + + // This is root of the documentation generation, but it's not directly + // responsible for generating markdown, it just constructs an AST that + // is the consumed by an emitter to actually write the files. + const documenter = new CustomMarkdownDocumenter({ + apiModel: newModel, + documenterConfig: { + outputTarget: 'markdown', + newlineKind: '\n', + // De ba dålig kod + configFilePath: '', + configFile: {}, + } as any, + outputFolder: outputDir, + }); + + // Clean up existing stuff and write ALL the docs! + await fs.remove(outputDir); + await fs.ensureDir(outputDir); + documenter.generateFiles(); +} diff --git a/packages/repo-tools/src/commands/api-reports/api-reports/createTemporaryTsConfig.ts b/packages/repo-tools/src/commands/api-reports/api-reports/createTemporaryTsConfig.ts new file mode 100644 index 0000000000..538fdd4ceb --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports/createTemporaryTsConfig.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { join } from 'path'; +import { paths as cliPaths } from '../../../lib/paths'; + +export async function createTemporaryTsConfig(includedPackageDirs: string[]) { + const path = cliPaths.resolveTargetRoot('tsconfig.tmp.json'); + + process.once('exit', () => { + fs.removeSync(path); + }); + + let assetTypeFile: string[] = []; + + try { + assetTypeFile = [ + require.resolve('@backstage/cli/asset-types/asset-types.d.ts'), + ]; + } catch { + /** ignore */ + } + + await fs.writeJson(path, { + extends: './tsconfig.json', + include: [ + // These two contain global definitions that are needed for stable API report generation + ...assetTypeFile, + ...includedPackageDirs.map(dir => join(dir, 'src')), + ], + // we don't exclude node_modules so that we can use the asset-types.d.ts file + exclude: [], + }); + + return path; +} diff --git a/packages/repo-tools/src/commands/api-reports/api-reports/index.ts b/packages/repo-tools/src/commands/api-reports/api-reports/index.ts new file mode 100644 index 0000000000..92945dc8f7 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runApiExtraction } from './runApiExtraction'; +export { buildDocs } from './buildDocs'; +export { createTemporaryTsConfig } from './createTemporaryTsConfig'; diff --git a/packages/repo-tools/src/commands/api-reports/api-reports/patchApiReportGeneration.ts b/packages/repo-tools/src/commands/api-reports/api-reports/patchApiReportGeneration.ts new file mode 100644 index 0000000000..4f25b9fd98 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports/patchApiReportGeneration.ts @@ -0,0 +1,173 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ExtractorMessage } from '@microsoft/api-extractor'; +import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; +import { Program } from 'typescript'; +import { paths as cliPaths } from '../../../lib/paths'; + +let applied = false; + +export function patchApiReportGeneration() { + // Make sure we only apply the patches once + if (applied) { + return; + } + applied = true; + + const { + ApiReportGenerator, + } = require('@microsoft/api-extractor/lib/generators/ApiReportGenerator'); + + function patchFileMessageFetcher( + router: any, + transform: (messages: ExtractorMessage[], ast?: AstDeclaration) => void, + ) { + const { + fetchAssociatedMessagesForReviewFile, + fetchUnassociatedMessagesForReviewFile, + } = router; + + router.fetchAssociatedMessagesForReviewFile = + function patchedFetchAssociatedMessagesForReviewFile( + ast: AstDeclaration | undefined, + ) { + const messages = fetchAssociatedMessagesForReviewFile.call(this, ast); + return transform(messages, ast); + }; + router.fetchUnassociatedMessagesForReviewFile = + function patchedFetchUnassociatedMessagesForReviewFile() { + const messages = fetchUnassociatedMessagesForReviewFile.call(this); + return transform(messages); + }; + } + + const originalGenerateReviewFileContent = + ApiReportGenerator.generateReviewFileContent; + ApiReportGenerator.generateReviewFileContent = + function decoratedGenerateReviewFileContent( + collector: { program: Program; messageRouter: any }, + ...moreArgs: any[] + ) { + const program = collector.program as Program; + // The purpose of this override is to allow the @ignore tag to be used to ignore warnings + // of the form "Warning: (ae-forgotten-export) The symbol "FooBar" needs to be exported by the entry point index.d.ts" + patchFileMessageFetcher( + collector.messageRouter, + (messages: ExtractorMessage[]) => { + return messages.filter(message => { + if (message.messageId !== 'ae-forgotten-export') { + return true; + } + + // Symbol name has to be extracted from the message :( + // There's frequently no AST for these exports because type literals + // aren't traversed by the generator. + const symbolMatch = message.text.match(/The symbol "([^"]+)"/); + if (!symbolMatch) { + throw new Error( + `Failed to extract symbol name from message "${message.text}"`, + ); + } + const [, symbolName] = symbolMatch; + + const sourceFile = + message.sourceFilePath && + program.getSourceFile(message.sourceFilePath); + if (!sourceFile) { + throw new Error( + `Failed to find source file in program at path "${message.sourceFilePath}"`, + ); + } + + // The local name of the symbol within the file, rather than the exported name + let localName = (sourceFile as any).identifiers?.get(symbolName); + + if (!localName) { + // Sometimes the symbol name is suffixed with a number to disambiguate, + // e.g. "Props_14" instead of "Props" if there are multiple Props interfaces + // so we try to strip that suffix and look up the symbol again. + const [, trimmedSymbolName] = symbolName.match(/(.*)_\d+/) || []; + localName = (sourceFile as any).identifiers?.get( + trimmedSymbolName, + ); + } + + if (!localName) { + throw new Error( + `Unable to find local name of "${symbolName}" in ${sourceFile.fileName}`, + ); + } + + // The local AST node of the export that we're missing + const local = (sourceFile as any).locals?.get(localName); + if (!local) { + return true; + } + + // Use the type checker to look up the actual declaration(s) rather than the one in the local file + const type = program + .getTypeChecker() + .getDeclaredTypeOfSymbol(local); + if (!type) { + throw new Error( + `Unable to find type declaration of "${symbolName}" in ${sourceFile.fileName}`, + ); + } + const declarations = type.aliasSymbol?.declarations; + if (!declarations || declarations.length === 0) { + return true; + } + + // If any of the TSDoc comments contain a @ignore tag, we ignore this message + const isIgnored = declarations.some(declaration => { + const tags = [(declaration as any).jsDoc] + .flat() + .filter(Boolean) + .flatMap((tagNode: any) => tagNode.tags); + + return tags.some(tag => tag?.tagName.text === 'ignore'); + }); + + return !isIgnored; + }); + }, + ); + + /** + * This monkey patching is for applying prettier to the API reports. This has to be patched into + * the middle of the process as API Extractor does a comparison of the contents of the old + * and new files during generation. This inserts the formatting just before that comparison. + */ + const content = originalGenerateReviewFileContent.call( + this, + collector, + ...moreArgs, + ); + + try { + const prettier = require('prettier') as typeof import('prettier'); + + const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {}; + return prettier.format(content, { + ...config, + parser: 'markdown', + }); + } catch (e) { + return content; + } + }; +} diff --git a/packages/repo-tools/src/commands/api-reports/api-reports/runApiExtraction.ts b/packages/repo-tools/src/commands/api-reports/api-reports/runApiExtraction.ts new file mode 100644 index 0000000000..1e4133def2 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports/runApiExtraction.ts @@ -0,0 +1,434 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CompilerState, + Extractor, + ExtractorConfig, + ExtractorLogLevel, +} from '@microsoft/api-extractor'; +import { TSDocTagSyntaxKind } from '@microsoft/tsdoc'; +import { TSDocConfigFile } from '@microsoft/tsdoc-config'; +import fs from 'fs-extra'; +import { groupBy } from 'lodash'; +import { minimatch } from 'minimatch'; +import { join, relative as relativePath, resolve as resolvePath } from 'path'; +import { getPackageExportDetails } from '../../../lib/getPackageExportDetails'; +import { paths as cliPaths } from '../../../lib/paths'; +import { logApiReportInstructions } from '../api-extractor'; +import { patchApiReportGeneration } from './patchApiReportGeneration'; + +const tmpDir = cliPaths.resolveTargetRoot( + './node_modules/.cache/api-extractor', +); + +export async function countApiReportWarnings(reportPath: string) { + try { + const content = await fs.readFile(reportPath, 'utf8'); + const lines = content.split('\n'); + + const lineWarnings = lines.filter(line => + line.includes('// Warning:'), + ).length; + + const trailerStart = lines.findIndex( + line => line === '// Warnings were encountered during analysis:', + ); + const trailerWarnings = + trailerStart === -1 + ? 0 + : lines.length - + trailerStart - + 4; /* 4 lines at the trailer and after are not warnings */ + + return lineWarnings + trailerWarnings; + } catch (error) { + if (error.code === 'ENOENT') { + return 0; + } + throw error; + } +} + +export async function getTsDocConfig() { + const tsdocConfigFile = await TSDocConfigFile.loadFile( + require.resolve('@microsoft/api-extractor/extends/tsdoc-base.json'), + ); + tsdocConfigFile.addTagDefinition({ + tagName: '@ignore', + syntaxKind: TSDocTagSyntaxKind.ModifierTag, + }); + tsdocConfigFile.addTagDefinition({ + tagName: '@config', + syntaxKind: TSDocTagSyntaxKind.BlockTag, + }); + tsdocConfigFile.setSupportForTag('@ignore', true); + tsdocConfigFile.setSupportForTag('@config', true); + return tsdocConfigFile; +} + +async function findPackageEntryPoints(packageDirs: string[]): Promise< + Array<{ + // package dir relative to root, e.g. "packages/backend-app-api" + packageDir: string; + // the name of the export, e.g. "index" or "alpha" + name: string; + // the path within the dist directory for this export, e.g. "alpha.d.ts" + distPath: string; + // the path within the dist-types directory of this package for this export, + // e.g. "src/entrypoints/foo/index.d.ts" + distTypesPath: string; + }> +> { + return Promise.all( + packageDirs.map(async packageDir => { + const pkg = await fs.readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); + + return getPackageExportDetails(pkg).map(details => { + return { packageDir, ...details }; + }); + }), + ).then(results => results.flat()); +} + +interface ApiExtractionOptions { + packageDirs: string[]; + outputDir: string; + isLocalBuild: boolean; + tsconfigFilePath: string; + allowWarnings?: boolean | string[]; + omitMessages?: string[]; + validateReleaseTags?: boolean; +} + +export async function runApiExtraction({ + packageDirs, + outputDir, + isLocalBuild, + tsconfigFilePath, + allowWarnings = false, + omitMessages = [], + validateReleaseTags = false, +}: ApiExtractionOptions) { + patchApiReportGeneration(); + + await fs.remove(outputDir); + + // The collection of all entry points of all packages, as a single list + const allEntryPoints = await findPackageEntryPoints(packageDirs); + + // The path (relative to the root) to ALL dist-types entry points (e.g. + // "dist-types/packages/backend-app-api/src/index.d.ts"). These are used as + // "extra"/contextual entry points for the extractor so that it can see the + // full context of things that are required by the local entry point being + // inspected. + const allDistTypesEntryPointPaths = allEntryPoints.map( + ({ packageDir, distTypesPath }) => { + return cliPaths.resolveTargetRoot( + './dist-types', + packageDir, + distTypesPath, + ); + }, + ); + + let compilerState: CompilerState | undefined = undefined; + + const allowWarningPkg = Array.isArray(allowWarnings) ? allowWarnings : []; + + const messagesConf: { [key: string]: { logLevel: string } } = {}; + for (const messageCode of omitMessages) { + messagesConf[messageCode] = { + logLevel: 'none', + }; + } + + const warnings = new Array(); + + for (const [packageDir, packageEntryPoints] of Object.entries( + groupBy(allEntryPoints, ep => ep.packageDir), + )) { + console.log(`## Processing ${packageDir}`); + const noBail = Array.isArray(allowWarnings) + ? allowWarnings.some(aw => aw === packageDir || minimatch(packageDir, aw)) + : allowWarnings; + + const projectFolder = cliPaths.resolveTargetRoot(packageDir); + const packageFolder = cliPaths.resolveTargetRoot( + './dist-types', + packageDir, + ); + + const remainingReportFiles = new Set( + fs.readdirSync(projectFolder).filter( + filename => + // https://regex101.com/r/QDZIV0/2 + filename !== 'knip-report.md' && + !filename.endsWith('.sql.md') && + // this has to temporarily match all old api report formats + filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/), + ), + ); + + for (const packageEntryPoint of packageEntryPoints) { + const suffix = + packageEntryPoint.name === 'index' ? '' : `-${packageEntryPoint.name}`; + const reportFileName = `report${suffix}`; + const reportPath = resolvePath(projectFolder, `${reportFileName}.api.md`); + + const warningCountBefore = await countApiReportWarnings(reportPath); + + const extractorConfig = ExtractorConfig.prepare({ + configObject: { + mainEntryPointFilePath: resolvePath( + packageFolder, + packageEntryPoint.distTypesPath, + ), + bundledPackages: [], + + compiler: { + tsconfigFilePath, + }, + + apiReport: { + enabled: true, + reportFileName, + reportFolder: projectFolder, + reportTempFolder: resolvePath( + outputDir, + `${suffix}`, + ), + }, + + docModel: { + // TODO(Rugvip): This skips docs for non-index entry points. We can try to work around it, but + // most likely it makes sense to wait for API Extractor to natively support exports. + enabled: packageEntryPoint.name === 'index', + apiJsonFilePath: resolvePath( + outputDir, + `${suffix}.api.json`, + ), + }, + + dtsRollup: { + enabled: false, + }, + + tsdocMetadata: { + enabled: false, + }, + + messages: { + // Silence compiler warnings, as these will prevent the CI build to work + compilerMessageReporting: { + default: { + logLevel: 'none' as ExtractorLogLevel.None, + // These contain absolute file paths, so can't be included in the report + // addToApiReportFile: true, + }, + }, + extractorMessageReporting: { + default: { + logLevel: 'warning' as ExtractorLogLevel.Warning, + addToApiReportFile: true, + }, + ...messagesConf, + }, + tsdocMessageReporting: { + default: { + logLevel: 'warning' as ExtractorLogLevel.Warning, + addToApiReportFile: true, + }, + }, + }, + + newlineKind: 'lf', + + projectFolder, + }, + configObjectFullPath: projectFolder, + packageJsonFullPath: resolvePath(projectFolder, 'package.json'), + tsdocConfigFile: await getTsDocConfig(), + ignoreMissingEntryPoint: true, + }); + + // remove extracted reports from current list + for (const reportConfig of extractorConfig.reportConfigs) { + remainingReportFiles.delete(reportConfig.fileName); + } + + // The `packageFolder` needs to point to the location within `dist-types` in order for relative + // paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`, + // which needs to point to the actual file, so we override `packageFolder` afterwards. + ( + extractorConfig as { + packageFolder: string; + } + ).packageFolder = packageFolder; + + if (!compilerState) { + compilerState = CompilerState.create(extractorConfig, { + additionalEntryPoints: allDistTypesEntryPointPaths, + }); + } + + // Message verbosity can't be configured, so just skip the check instead + (Extractor as any)._checkCompilerCompatibility = () => {}; + + let shouldLogInstructions = false; + let conflictingFile: undefined | string = undefined; + + // Invoke API Extractor + const extractorResult = Extractor.invoke(extractorConfig, { + localBuild: isLocalBuild, + showVerboseMessages: false, + showDiagnostics: false, + messageCallback(message) { + if (message.text.includes('The API report file is missing')) { + shouldLogInstructions = true; + } + + // Detect messages like the following being output by the generator: + // Warning: You have changed the API signature for this project. Please copy the file "/home/runner/work/backstage/backstage/node_modules/.cache/api-extractor/backend-test-utils/report.api.md" to "report.api.md", or perform a local build (which does this automatically). See the Git repo documentation for more info. + if ( + message.text.includes( + 'You have changed the API signature for this project.', + ) + ) { + shouldLogInstructions = true; + const match = message.text.match( + /Please copy the file "(.*)" to "report\.api\.md"/, + ); + if (match) { + conflictingFile = match[1]; + } + } + }, + compilerState, + }); + + // This release tag validation makes sure that the release tag of known entry points match expectations. + // The root index entry point is only allowed @public exports, while /alpha and /beta only allow @alpha and @beta. + if ( + validateReleaseTags && + fs.pathExistsSync(extractorConfig.reportFilePath) + ) { + if (['index', 'alpha', 'beta'].includes(packageEntryPoint.name)) { + const report = await fs.readFile( + extractorConfig.reportFilePath, + 'utf8', + ); + const lines = report.split(/\r?\n/); + const expectedTag = + packageEntryPoint.name === 'index' + ? 'public' + : packageEntryPoint.name; + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]; + const match = line.match(/^\/\/ @(alpha|beta|public)/); + if (match && match[1] !== expectedTag) { + // Because of limitations in the type script rollup logic we need to allow public exports from the other release stages + // TODO(Rugvip): Try to work around the need for this exception + if (expectedTag !== 'public' && match[1] === 'public') { + continue; + } + throw new Error( + `Unexpected release tag ${match[1]} in ${ + extractorConfig.reportFilePath + } at line ${i + 1}`, + ); + } + } + } + } + + if (!extractorResult.succeeded) { + if (shouldLogInstructions) { + logApiReportInstructions(); + + if (conflictingFile) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + tmpDir, + conflictingFile, + )}, with the following content:`, + ); + console.log(''); + + const content = await fs.readFile(conflictingFile, 'utf8'); + console.log(content); + + logApiReportInstructions(); + } + } + + throw new Error( + `API Extractor completed with ${extractorResult.errorCount} errors` + + ` and ${extractorResult.warningCount} warnings`, + ); + } + + const warningCountAfter = await countApiReportWarnings(reportPath); + + if (noBail) { + console.log(`Skipping warnings check for ${packageDir}`); + } + if (warningCountAfter > 0 && !noBail) { + throw new Error( + `The API Report for ${packageDir} is not allowed to have warnings`, + ); + } + if (warningCountAfter === 0 && allowWarningPkg.includes(packageDir)) { + console.log( + `No need to allow warnings for ${packageDir}, it does not have any`, + ); + } + if (warningCountAfter > warningCountBefore) { + warnings.push( + `The API Report for ${packageDir} introduces new warnings. ` + + 'Please fix these warnings in order to keep the API Reports tidy.', + ); + } + } + + if (remainingReportFiles.size > 0) { + if (isLocalBuild) { + for (const f of remainingReportFiles) { + fs.rmSync(resolvePath(projectFolder, f)); + console.log(`Deleted deprecated API report ${f}`); + } + } else { + const staleList = [...remainingReportFiles] + .map(f => join(packageDir, f)) + .join(', '); + throw new Error( + `The API Report(s) ${staleList} are no longer relevant and should be deleted`, + ); + } + } + } + + if (warnings.length > 0) { + console.warn(); + for (const warning of warnings) { + console.warn(warning); + } + console.warn(); + } +}