repo-tools: refactor api report generation

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2024-12-29 12:11:32 +01:00
parent f5d1686b17
commit c8ddc04f5b
8 changed files with 1293 additions and 1186 deletions
File diff suppressed because it is too large Load Diff
@@ -18,16 +18,13 @@ import { createMockDirectory } from '@backstage/backend-test-utils';
import { normalize } from 'path';
import * as pathsLib from '../../lib/paths';
import {
buildDocs,
categorizePackageDirs,
runApiExtraction,
} from './api-extractor';
import { categorizePackageDirs } from './api-extractor';
import { buildApiReports } from './api-reports';
import { generateTypeDeclarations } from './generateTypeDeclarations';
import { PackageGraph } from '@backstage/cli-node';
import { runCliExtraction } from './cli-reports';
import { runApiExtraction, buildDocs } from './api-reports/index';
jest.mock('./generateTypeDeclarations');
// create mocks for the dependencies of the `buildApiReports` function
@@ -15,16 +15,16 @@
*/
import { OptionValues } from 'commander';
import {
buildDocs,
categorizePackageDirs,
createTemporaryTsConfig,
runApiExtraction,
} from './api-extractor';
import { categorizePackageDirs } from './api-extractor';
import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
import { generateTypeDeclarations } from './generateTypeDeclarations';
import { runSqlExtraction } from './sql-reports';
import { runCliExtraction } from './cli-reports';
import {
runApiExtraction,
buildDocs,
createTemporaryTsConfig,
} from './api-reports/index';
type Options = {
ci?: boolean;
@@ -0,0 +1,609 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolve as resolvePath } from 'path';
import fs from 'fs-extra';
import {
DocBlockTag,
DocLinkTag,
DocNode,
DocPlainText,
IDocNodeContainerParameters,
Standardization,
TSDocConfiguration,
TSDocTagSyntaxKind,
} from '@microsoft/tsdoc';
import {
ApiItem,
ApiItemKind,
ApiModel,
ApiPackage,
} from '@microsoft/api-extractor-model';
import {
IMarkdownDocumenterOptions,
MarkdownDocumenter,
} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter';
import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable';
import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow';
import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading';
import {
CustomMarkdownEmitter,
ICustomMarkdownEmitterOptions,
} from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter';
import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter';
/*
WARNING: Bring a blanket if you're gonna read the code below
There's some weird shit going on here, and it's because we cba
forking rushstack to modify the api-documenter markdown generation,
which otherwise is the recommended way to do customizations.
*/
type ExcerptToken = {
kind: string;
text: string;
canonicalReference?: string;
};
class ExcerptTokenMatcher {
readonly #tokens: ExcerptToken[];
constructor(tokens: ExcerptToken[]) {
this.#tokens = tokens.slice();
}
nextContent() {
const token = this.#tokens.shift();
if (token?.kind === 'Content') {
return token.text;
}
return undefined;
}
matchContent(expectedText: string) {
const text = this.nextContent();
return text !== expectedText;
}
getTokensUntilArrow() {
const tokens = [];
for (;;) {
const token = this.#tokens.shift();
if (token === undefined) {
return undefined;
}
if (token.kind === 'Content' && token.text === ') => ') {
return tokens;
}
tokens.push(token);
}
}
getComponentReturnTokens() {
const first = this.#tokens.shift();
if (!first) {
return undefined;
}
const second = this.#tokens.shift();
if (this.#tokens.length !== 0) {
return undefined;
}
if (first.kind !== 'Reference' || first.text !== 'JSX.Element') {
return undefined;
}
if (!second) {
return [first];
} else if (second.kind === 'Content' && second.text === ' | null') {
return [first, second];
}
return undefined;
}
}
class ApiModelTransforms {
static deserializeWithTransforms(
serialized: any,
transforms: Array<(member: any) => any>,
): ApiPackage {
if (serialized.kind !== 'Package') {
throw new Error(
`Unexpected root kind in serialized ApiPackage, ${serialized.kind}`,
);
}
if (serialized.members.length !== 1) {
throw new Error(
`Unexpected members in serialized ApiPackage, [${serialized.members
.map((m: { kind: any }) => m.kind)
.join(' ')}]`,
);
}
const [entryPoint] = serialized.members;
if (entryPoint.kind !== 'EntryPoint') {
throw new Error(
`Unexpected kind in serialized ApiPackage member, ${entryPoint.kind}`,
);
}
const transformed = {
...serialized,
members: [
{
...entryPoint,
members: entryPoint.members.map((member: any) =>
transforms.reduce((m, t) => t(m), member),
),
},
],
};
return ApiPackage.deserialize(
transformed,
transformed.metadata,
) as ApiPackage;
}
static transformArrowComponents = (member: any) => {
if (member.kind !== 'Variable') {
return member;
}
const { name, excerptTokens } = member;
// First letter in name must be uppercase
const [firstChar] = name;
if (firstChar.toLocaleUpperCase('en-US') !== firstChar) {
return member;
}
// First content must match expected declaration format
const tokens = new ExcerptTokenMatcher(excerptTokens);
if (tokens.nextContent() !== `${name}: `) {
return member;
}
// Next needs to be an arrow with `props` parameters or no parameters
// followed by a return type of `JSX.Element | null` or just `JSX.Element`
const declStart = tokens.nextContent();
if (declStart === '(props: ' || declStart === '(_props: ') {
const props = tokens.getTokensUntilArrow();
const ret = tokens.getComponentReturnTokens();
if (props && ret) {
return this.makeComponentMember(member, ret, props);
}
} else if (declStart === '() => ') {
const ret = tokens.getComponentReturnTokens();
if (ret) {
return this.makeComponentMember(member, ret);
}
}
return member;
};
static makeComponentMember(
member: any,
ret: ExcerptToken[],
props?: ExcerptToken[],
) {
const declTokens = props
? [
{
kind: 'Content',
text: `export declare function ${member.name}(props: `,
},
...props,
{
kind: 'Content',
text: '): ',
},
]
: [
{
kind: 'Content',
text: `export declare function ${member.name}(): `,
},
];
return {
kind: 'Function',
name: member.name,
releaseTag: member.releaseTag,
docComment: member.docComment ?? '',
canonicalReference: member.canonicalReference,
excerptTokens: [...declTokens, ...ret],
returnTypeTokenRange: {
startIndex: declTokens.length,
endIndex: declTokens.length + ret.length,
},
parameters: props
? [
{
parameterName: 'props',
parameterTypeTokenRange: {
startIndex: 1,
endIndex: 1 + props.length,
},
},
]
: [],
overloadIndex: 1,
};
}
static transformTrimDeclare = (member: any) => {
const { excerptTokens } = member;
const firstContent = new ExcerptTokenMatcher(excerptTokens).nextContent();
if (firstContent && firstContent.startsWith('export declare ')) {
return {
...member,
excerptTokens: [
{
kind: 'Content',
text: firstContent.slice('export declare '.length),
},
...excerptTokens.slice(1),
],
};
}
return member;
};
}
export async function buildDocs({
inputDir,
outputDir,
}: {
inputDir: string;
outputDir: string;
}) {
// We start by constructing our own model from the files so that
// we get a change to modify them, as the model is otherwise read-only.
const parseFile = async (filename: string): Promise<any> => {
console.log(`Reading ${filename}`);
return fs.readJson(resolvePath(inputDir, filename));
};
const filenames = await fs.readdir(inputDir);
const serializedPackages = await Promise.all(
filenames
.filter(filename => filename.match(/\.api\.json$/i))
.map(parseFile),
);
const newModel = new ApiModel();
for (const serialized of serializedPackages) {
newModel.addMember(
ApiModelTransforms.deserializeWithTransforms(serialized, [
ApiModelTransforms.transformArrowComponents,
ApiModelTransforms.transformTrimDeclare,
]),
);
}
// The doc AST need to be extended with custom nodes if we want to
// add any extra content.
// This one is for the YAML front matter that we need for the microsite.
class DocFrontMatter extends DocNode {
static kind = 'DocFrontMatter';
public readonly values: { [name: string]: unknown };
public constructor(
parameters: IDocNodeContainerParameters & {
values: { [name: string]: unknown };
},
) {
super(parameters);
this.values = parameters.values;
}
/** @override */
public get kind(): string {
return DocFrontMatter.kind;
}
}
// This class only propose is to have a different kind and be able to render links with backticks
class DocCodeSpanLink extends DocLinkTag {
static kind = 'DocCodeSpanLink';
/** @override */
public get kind(): string {
return DocCodeSpanLink.kind;
}
}
// This is where we actually write the markdown and where we can hook
// in the rendering of our own nodes.
class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter {
// Until https://github.com/microsoft/rushstack/issues/2914 gets fixed or we change markdown renderer we need a fix
// to render pipe | character correctly.
protected getEscapedText(text: string): string {
return text
.replace(/\\/g, '\\\\') // first replace the escape character
.replace(/[*#[\]_`~]/g, x => `\\${x}`) // then escape any special characters
.replace(/---/g, '\\-\\-\\-') // hyphens only if it's 3 or more
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/\|/g, '&#124;');
}
/** @override */
protected writeNode(
docNode: DocNode,
context: IMarkdownEmitterContext<ICustomMarkdownEmitterOptions>,
docNodeSiblings: boolean,
): void {
switch (docNode.kind) {
case DocFrontMatter.kind: {
const node = docNode as DocFrontMatter;
context.writer.writeLine('---');
for (const [name, value] of Object.entries(node.values)) {
if (value) {
context.writer.writeLine(
`${name}: "${String(value).replace(/\"/g, '')}"`,
);
}
}
context.writer.writeLine('---');
context.writer.writeLine();
break;
}
case 'BlockTag': {
const node = docNode as DocBlockTag;
if (node.tagName === '@config') {
context.writer.writeLine('## Related config ');
}
break;
}
case DocCodeSpanLink.kind: {
const node = docNode as DocLinkTag;
if (node.codeDestination) {
// TODO @sarabadu understand if we need `codeDestination` at all on this custom DocCodeSpanLink
super.writeLinkTagWithCodeDestination(node, context);
} else if (node.urlDestination) {
const linkText =
node.linkText !== undefined ? node.linkText : node.urlDestination;
const encodedLinkText = this.getEscapedText(
linkText.replace(/\s+/g, ' '),
);
context.writer.write('[');
context.writer.write(`\`${encodedLinkText}\``);
context.writer.write(`](${node.urlDestination})`);
} else if (node.linkText) {
this.writePlainText(node.linkText, context);
}
break;
}
default:
super.writeNode(docNode, context, docNodeSiblings);
}
}
/** @override */
emit(
stringBuilder: any,
docNode: DocNode,
options: ICustomMarkdownEmitterOptions,
) {
// Hack to get rid of the leading comment of each file, since
// we want the front matter to come first
stringBuilder._chunks.length = 0;
return super.emit(stringBuilder, docNode, options);
}
}
class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) {
constructor(options: IMarkdownDocumenterOptions) {
super(options);
// It's a strict model, we gotta register the allowed usage of our new node
(
this._tsdocConfiguration as TSDocConfiguration
).docNodeManager.registerDocNodes('@backstage/docs', [
{ docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter },
]);
(
this._tsdocConfiguration as TSDocConfiguration
).docNodeManager.registerDocNodes('@backstage/docs', [
{ docNodeKind: DocCodeSpanLink.kind, constructor: DocCodeSpanLink },
]);
(
this._tsdocConfiguration as TSDocConfiguration
).docNodeManager.registerAllowableChildren('Paragraph', [
DocFrontMatter.kind,
DocCodeSpanLink.kind,
]);
const def = {
tagName: '@config',
syntaxKind: TSDocTagSyntaxKind.BlockTag,
tagNameWithUpperCase: '@CONFIG',
standardization: Standardization.Extended,
allowMultiple: false,
};
(this._tsdocConfiguration as TSDocConfiguration).addTagDefinition(def);
(this._tsdocConfiguration as TSDocConfiguration).setSupportForTag(
def,
true,
);
this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel);
}
private _getFilenameForApiItem(apiItem: ApiItem): string {
const filename: string = super._getFilenameForApiItem(apiItem);
if (filename.includes('.html.')) {
return filename.replace(/\.html\./g, '._html.');
}
return filename;
}
// We don't really get many chances to modify the generated AST
// so we hook in wherever we can. In this case we add the front matter
// just before writing the breadcrumbs at the top.
/** @override */
_writeBreadcrumb(output: any, apiItem: ApiItem & { name: string }) {
let title;
let description;
const name = apiItem.getScopedNameWithinPackage();
if (name) {
title = name;
description = `API reference for ${apiItem.getScopedNameWithinPackage()}`;
} else if (apiItem.kind === 'Model') {
title = 'Package Index';
description = 'Index of all Backstage Packages';
} else if (apiItem.name) {
title = apiItem.name;
description = `API Reference for ${apiItem.name}`;
} else {
title = apiItem.displayName;
description = `API Reference for ${apiItem.displayName}`;
}
// Add our front matter
output.appendNodeInParagraph(
new DocFrontMatter({
configuration: this._tsdocConfiguration,
values: {
id: this._getFilenameForApiItem(apiItem).slice(0, -3),
title,
description,
},
}),
);
const configuration: TSDocConfiguration = this._tsdocConfiguration;
output.appendNodeInParagraph(
new DocLinkTag({
configuration,
tagName: '@link',
linkText: 'Home',
urlDestination: this._getLinkFilenameForApiItem(this._apiModel),
}),
);
for (const hierarchyItem of apiItem.getHierarchy()) {
switch (hierarchyItem.kind) {
case ApiItemKind.Model:
case ApiItemKind.EntryPoint:
// We don't show the model as part of the breadcrumb because it is the root-level container.
// We don't show the entry point because today API Extractor doesn't support multiple entry points;
// this may change in the future.
break;
default:
output.appendNodesInParagraph([
new DocPlainText({
configuration,
text: ' > ',
}),
new DocCodeSpanLink({
configuration,
tagName: '@link',
linkText: hierarchyItem.displayName,
urlDestination: this._getLinkFilenameForApiItem(hierarchyItem),
}),
]);
}
}
// We wanna ignore the header that always gets written after the breadcrumb
// This otherwise becomes more or less a duplicate of the title in the front matter
const oldAppendNode = output.appendNode;
output.appendNode = () => {
output.appendNode = oldAppendNode;
};
}
_writeModelTable(
output: { appendNode: (arg0: DocTable | DocHeading) => void },
apiModel: { members: any },
): void {
const configuration = this._tsdocConfiguration;
const packagesTable = new DocTable({
configuration,
headerTitles: ['Package', 'Description'],
});
const pluginsTable = new DocTable({
configuration,
headerTitles: ['Package', 'Description'],
});
for (const apiMember of apiModel.members) {
const row = new DocTableRow({ configuration }, [
this._createTitleCell(apiMember),
this._createDescriptionCell(apiMember),
]);
if (apiMember.kind === 'Package') {
this._writeApiItemPage(apiMember);
if (apiMember.name.startsWith('@backstage/plugin-')) {
pluginsTable.addRow(row);
} else {
packagesTable.addRow(row);
}
}
}
if (packagesTable.rows.length > 0) {
output.appendNode(
new DocHeading({
configuration: this._tsdocConfiguration,
title: 'Packages',
}),
);
output.appendNode(packagesTable);
}
if (pluginsTable.rows.length > 0) {
output.appendNode(
new DocHeading({
configuration: this._tsdocConfiguration,
title: 'Plugins',
}),
);
output.appendNode(pluginsTable);
}
}
}
// This is root of the documentation generation, but it's not directly
// responsible for generating markdown, it just constructs an AST that
// is the consumed by an emitter to actually write the files.
const documenter = new CustomMarkdownDocumenter({
apiModel: newModel,
documenterConfig: {
outputTarget: 'markdown',
newlineKind: '\n',
// De ba dålig kod
configFilePath: '',
configFile: {},
} as any,
outputFolder: outputDir,
});
// Clean up existing stuff and write ALL the docs!
await fs.remove(outputDir);
await fs.ensureDir(outputDir);
documenter.generateFiles();
}
@@ -0,0 +1,50 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { join } from 'path';
import { paths as cliPaths } from '../../../lib/paths';
export async function createTemporaryTsConfig(includedPackageDirs: string[]) {
const path = cliPaths.resolveTargetRoot('tsconfig.tmp.json');
process.once('exit', () => {
fs.removeSync(path);
});
let assetTypeFile: string[] = [];
try {
assetTypeFile = [
require.resolve('@backstage/cli/asset-types/asset-types.d.ts'),
];
} catch {
/** ignore */
}
await fs.writeJson(path, {
extends: './tsconfig.json',
include: [
// These two contain global definitions that are needed for stable API report generation
...assetTypeFile,
...includedPackageDirs.map(dir => join(dir, 'src')),
],
// we don't exclude node_modules so that we can use the asset-types.d.ts file
exclude: [],
});
return path;
}
@@ -0,0 +1,19 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { runApiExtraction } from './runApiExtraction';
export { buildDocs } from './buildDocs';
export { createTemporaryTsConfig } from './createTemporaryTsConfig';
@@ -0,0 +1,173 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ExtractorMessage } from '@microsoft/api-extractor';
import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration';
import { Program } from 'typescript';
import { paths as cliPaths } from '../../../lib/paths';
let applied = false;
export function patchApiReportGeneration() {
// Make sure we only apply the patches once
if (applied) {
return;
}
applied = true;
const {
ApiReportGenerator,
} = require('@microsoft/api-extractor/lib/generators/ApiReportGenerator');
function patchFileMessageFetcher(
router: any,
transform: (messages: ExtractorMessage[], ast?: AstDeclaration) => void,
) {
const {
fetchAssociatedMessagesForReviewFile,
fetchUnassociatedMessagesForReviewFile,
} = router;
router.fetchAssociatedMessagesForReviewFile =
function patchedFetchAssociatedMessagesForReviewFile(
ast: AstDeclaration | undefined,
) {
const messages = fetchAssociatedMessagesForReviewFile.call(this, ast);
return transform(messages, ast);
};
router.fetchUnassociatedMessagesForReviewFile =
function patchedFetchUnassociatedMessagesForReviewFile() {
const messages = fetchUnassociatedMessagesForReviewFile.call(this);
return transform(messages);
};
}
const originalGenerateReviewFileContent =
ApiReportGenerator.generateReviewFileContent;
ApiReportGenerator.generateReviewFileContent =
function decoratedGenerateReviewFileContent(
collector: { program: Program; messageRouter: any },
...moreArgs: any[]
) {
const program = collector.program as Program;
// The purpose of this override is to allow the @ignore tag to be used to ignore warnings
// of the form "Warning: (ae-forgotten-export) The symbol "FooBar" needs to be exported by the entry point index.d.ts"
patchFileMessageFetcher(
collector.messageRouter,
(messages: ExtractorMessage[]) => {
return messages.filter(message => {
if (message.messageId !== 'ae-forgotten-export') {
return true;
}
// Symbol name has to be extracted from the message :(
// There's frequently no AST for these exports because type literals
// aren't traversed by the generator.
const symbolMatch = message.text.match(/The symbol "([^"]+)"/);
if (!symbolMatch) {
throw new Error(
`Failed to extract symbol name from message "${message.text}"`,
);
}
const [, symbolName] = symbolMatch;
const sourceFile =
message.sourceFilePath &&
program.getSourceFile(message.sourceFilePath);
if (!sourceFile) {
throw new Error(
`Failed to find source file in program at path "${message.sourceFilePath}"`,
);
}
// The local name of the symbol within the file, rather than the exported name
let localName = (sourceFile as any).identifiers?.get(symbolName);
if (!localName) {
// Sometimes the symbol name is suffixed with a number to disambiguate,
// e.g. "Props_14" instead of "Props" if there are multiple Props interfaces
// so we try to strip that suffix and look up the symbol again.
const [, trimmedSymbolName] = symbolName.match(/(.*)_\d+/) || [];
localName = (sourceFile as any).identifiers?.get(
trimmedSymbolName,
);
}
if (!localName) {
throw new Error(
`Unable to find local name of "${symbolName}" in ${sourceFile.fileName}`,
);
}
// The local AST node of the export that we're missing
const local = (sourceFile as any).locals?.get(localName);
if (!local) {
return true;
}
// Use the type checker to look up the actual declaration(s) rather than the one in the local file
const type = program
.getTypeChecker()
.getDeclaredTypeOfSymbol(local);
if (!type) {
throw new Error(
`Unable to find type declaration of "${symbolName}" in ${sourceFile.fileName}`,
);
}
const declarations = type.aliasSymbol?.declarations;
if (!declarations || declarations.length === 0) {
return true;
}
// If any of the TSDoc comments contain a @ignore tag, we ignore this message
const isIgnored = declarations.some(declaration => {
const tags = [(declaration as any).jsDoc]
.flat()
.filter(Boolean)
.flatMap((tagNode: any) => tagNode.tags);
return tags.some(tag => tag?.tagName.text === 'ignore');
});
return !isIgnored;
});
},
);
/**
* This monkey patching is for applying prettier to the API reports. This has to be patched into
* the middle of the process as API Extractor does a comparison of the contents of the old
* and new files during generation. This inserts the formatting just before that comparison.
*/
const content = originalGenerateReviewFileContent.call(
this,
collector,
...moreArgs,
);
try {
const prettier = require('prettier') as typeof import('prettier');
const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {};
return prettier.format(content, {
...config,
parser: 'markdown',
});
} catch (e) {
return content;
}
};
}
@@ -0,0 +1,434 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
CompilerState,
Extractor,
ExtractorConfig,
ExtractorLogLevel,
} from '@microsoft/api-extractor';
import { TSDocTagSyntaxKind } from '@microsoft/tsdoc';
import { TSDocConfigFile } from '@microsoft/tsdoc-config';
import fs from 'fs-extra';
import { groupBy } from 'lodash';
import { minimatch } from 'minimatch';
import { join, relative as relativePath, resolve as resolvePath } from 'path';
import { getPackageExportDetails } from '../../../lib/getPackageExportDetails';
import { paths as cliPaths } from '../../../lib/paths';
import { logApiReportInstructions } from '../api-extractor';
import { patchApiReportGeneration } from './patchApiReportGeneration';
const tmpDir = cliPaths.resolveTargetRoot(
'./node_modules/.cache/api-extractor',
);
export async function countApiReportWarnings(reportPath: string) {
try {
const content = await fs.readFile(reportPath, 'utf8');
const lines = content.split('\n');
const lineWarnings = lines.filter(line =>
line.includes('// Warning:'),
).length;
const trailerStart = lines.findIndex(
line => line === '// Warnings were encountered during analysis:',
);
const trailerWarnings =
trailerStart === -1
? 0
: lines.length -
trailerStart -
4; /* 4 lines at the trailer and after are not warnings */
return lineWarnings + trailerWarnings;
} catch (error) {
if (error.code === 'ENOENT') {
return 0;
}
throw error;
}
}
export async function getTsDocConfig() {
const tsdocConfigFile = await TSDocConfigFile.loadFile(
require.resolve('@microsoft/api-extractor/extends/tsdoc-base.json'),
);
tsdocConfigFile.addTagDefinition({
tagName: '@ignore',
syntaxKind: TSDocTagSyntaxKind.ModifierTag,
});
tsdocConfigFile.addTagDefinition({
tagName: '@config',
syntaxKind: TSDocTagSyntaxKind.BlockTag,
});
tsdocConfigFile.setSupportForTag('@ignore', true);
tsdocConfigFile.setSupportForTag('@config', true);
return tsdocConfigFile;
}
async function findPackageEntryPoints(packageDirs: string[]): Promise<
Array<{
// package dir relative to root, e.g. "packages/backend-app-api"
packageDir: string;
// the name of the export, e.g. "index" or "alpha"
name: string;
// the path within the dist directory for this export, e.g. "alpha.d.ts"
distPath: string;
// the path within the dist-types directory of this package for this export,
// e.g. "src/entrypoints/foo/index.d.ts"
distTypesPath: string;
}>
> {
return Promise.all(
packageDirs.map(async packageDir => {
const pkg = await fs.readJson(
cliPaths.resolveTargetRoot(packageDir, 'package.json'),
);
return getPackageExportDetails(pkg).map(details => {
return { packageDir, ...details };
});
}),
).then(results => results.flat());
}
interface ApiExtractionOptions {
packageDirs: string[];
outputDir: string;
isLocalBuild: boolean;
tsconfigFilePath: string;
allowWarnings?: boolean | string[];
omitMessages?: string[];
validateReleaseTags?: boolean;
}
export async function runApiExtraction({
packageDirs,
outputDir,
isLocalBuild,
tsconfigFilePath,
allowWarnings = false,
omitMessages = [],
validateReleaseTags = false,
}: ApiExtractionOptions) {
patchApiReportGeneration();
await fs.remove(outputDir);
// The collection of all entry points of all packages, as a single list
const allEntryPoints = await findPackageEntryPoints(packageDirs);
// The path (relative to the root) to ALL dist-types entry points (e.g.
// "dist-types/packages/backend-app-api/src/index.d.ts"). These are used as
// "extra"/contextual entry points for the extractor so that it can see the
// full context of things that are required by the local entry point being
// inspected.
const allDistTypesEntryPointPaths = allEntryPoints.map(
({ packageDir, distTypesPath }) => {
return cliPaths.resolveTargetRoot(
'./dist-types',
packageDir,
distTypesPath,
);
},
);
let compilerState: CompilerState | undefined = undefined;
const allowWarningPkg = Array.isArray(allowWarnings) ? allowWarnings : [];
const messagesConf: { [key: string]: { logLevel: string } } = {};
for (const messageCode of omitMessages) {
messagesConf[messageCode] = {
logLevel: 'none',
};
}
const warnings = new Array<string>();
for (const [packageDir, packageEntryPoints] of Object.entries(
groupBy(allEntryPoints, ep => ep.packageDir),
)) {
console.log(`## Processing ${packageDir}`);
const noBail = Array.isArray(allowWarnings)
? allowWarnings.some(aw => aw === packageDir || minimatch(packageDir, aw))
: allowWarnings;
const projectFolder = cliPaths.resolveTargetRoot(packageDir);
const packageFolder = cliPaths.resolveTargetRoot(
'./dist-types',
packageDir,
);
const remainingReportFiles = new Set(
fs.readdirSync(projectFolder).filter(
filename =>
// https://regex101.com/r/QDZIV0/2
filename !== 'knip-report.md' &&
!filename.endsWith('.sql.md') &&
// this has to temporarily match all old api report formats
filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/),
),
);
for (const packageEntryPoint of packageEntryPoints) {
const suffix =
packageEntryPoint.name === 'index' ? '' : `-${packageEntryPoint.name}`;
const reportFileName = `report${suffix}`;
const reportPath = resolvePath(projectFolder, `${reportFileName}.api.md`);
const warningCountBefore = await countApiReportWarnings(reportPath);
const extractorConfig = ExtractorConfig.prepare({
configObject: {
mainEntryPointFilePath: resolvePath(
packageFolder,
packageEntryPoint.distTypesPath,
),
bundledPackages: [],
compiler: {
tsconfigFilePath,
},
apiReport: {
enabled: true,
reportFileName,
reportFolder: projectFolder,
reportTempFolder: resolvePath(
outputDir,
`<unscopedPackageName>${suffix}`,
),
},
docModel: {
// TODO(Rugvip): This skips docs for non-index entry points. We can try to work around it, but
// most likely it makes sense to wait for API Extractor to natively support exports.
enabled: packageEntryPoint.name === 'index',
apiJsonFilePath: resolvePath(
outputDir,
`<unscopedPackageName>${suffix}.api.json`,
),
},
dtsRollup: {
enabled: false,
},
tsdocMetadata: {
enabled: false,
},
messages: {
// Silence compiler warnings, as these will prevent the CI build to work
compilerMessageReporting: {
default: {
logLevel: 'none' as ExtractorLogLevel.None,
// These contain absolute file paths, so can't be included in the report
// addToApiReportFile: true,
},
},
extractorMessageReporting: {
default: {
logLevel: 'warning' as ExtractorLogLevel.Warning,
addToApiReportFile: true,
},
...messagesConf,
},
tsdocMessageReporting: {
default: {
logLevel: 'warning' as ExtractorLogLevel.Warning,
addToApiReportFile: true,
},
},
},
newlineKind: 'lf',
projectFolder,
},
configObjectFullPath: projectFolder,
packageJsonFullPath: resolvePath(projectFolder, 'package.json'),
tsdocConfigFile: await getTsDocConfig(),
ignoreMissingEntryPoint: true,
});
// remove extracted reports from current list
for (const reportConfig of extractorConfig.reportConfigs) {
remainingReportFiles.delete(reportConfig.fileName);
}
// The `packageFolder` needs to point to the location within `dist-types` in order for relative
// paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`,
// which needs to point to the actual file, so we override `packageFolder` afterwards.
(
extractorConfig as {
packageFolder: string;
}
).packageFolder = packageFolder;
if (!compilerState) {
compilerState = CompilerState.create(extractorConfig, {
additionalEntryPoints: allDistTypesEntryPointPaths,
});
}
// Message verbosity can't be configured, so just skip the check instead
(Extractor as any)._checkCompilerCompatibility = () => {};
let shouldLogInstructions = false;
let conflictingFile: undefined | string = undefined;
// Invoke API Extractor
const extractorResult = Extractor.invoke(extractorConfig, {
localBuild: isLocalBuild,
showVerboseMessages: false,
showDiagnostics: false,
messageCallback(message) {
if (message.text.includes('The API report file is missing')) {
shouldLogInstructions = true;
}
// Detect messages like the following being output by the generator:
// Warning: You have changed the API signature for this project. Please copy the file "/home/runner/work/backstage/backstage/node_modules/.cache/api-extractor/backend-test-utils/report.api.md" to "report.api.md", or perform a local build (which does this automatically). See the Git repo documentation for more info.
if (
message.text.includes(
'You have changed the API signature for this project.',
)
) {
shouldLogInstructions = true;
const match = message.text.match(
/Please copy the file "(.*)" to "report\.api\.md"/,
);
if (match) {
conflictingFile = match[1];
}
}
},
compilerState,
});
// This release tag validation makes sure that the release tag of known entry points match expectations.
// The root index entry point is only allowed @public exports, while /alpha and /beta only allow @alpha and @beta.
if (
validateReleaseTags &&
fs.pathExistsSync(extractorConfig.reportFilePath)
) {
if (['index', 'alpha', 'beta'].includes(packageEntryPoint.name)) {
const report = await fs.readFile(
extractorConfig.reportFilePath,
'utf8',
);
const lines = report.split(/\r?\n/);
const expectedTag =
packageEntryPoint.name === 'index'
? 'public'
: packageEntryPoint.name;
for (let i = 0; i < lines.length; i += 1) {
const line = lines[i];
const match = line.match(/^\/\/ @(alpha|beta|public)/);
if (match && match[1] !== expectedTag) {
// Because of limitations in the type script rollup logic we need to allow public exports from the other release stages
// TODO(Rugvip): Try to work around the need for this exception
if (expectedTag !== 'public' && match[1] === 'public') {
continue;
}
throw new Error(
`Unexpected release tag ${match[1]} in ${
extractorConfig.reportFilePath
} at line ${i + 1}`,
);
}
}
}
}
if (!extractorResult.succeeded) {
if (shouldLogInstructions) {
logApiReportInstructions();
if (conflictingFile) {
console.log('');
console.log(
`The conflicting file is ${relativePath(
tmpDir,
conflictingFile,
)}, with the following content:`,
);
console.log('');
const content = await fs.readFile(conflictingFile, 'utf8');
console.log(content);
logApiReportInstructions();
}
}
throw new Error(
`API Extractor completed with ${extractorResult.errorCount} errors` +
` and ${extractorResult.warningCount} warnings`,
);
}
const warningCountAfter = await countApiReportWarnings(reportPath);
if (noBail) {
console.log(`Skipping warnings check for ${packageDir}`);
}
if (warningCountAfter > 0 && !noBail) {
throw new Error(
`The API Report for ${packageDir} is not allowed to have warnings`,
);
}
if (warningCountAfter === 0 && allowWarningPkg.includes(packageDir)) {
console.log(
`No need to allow warnings for ${packageDir}, it does not have any`,
);
}
if (warningCountAfter > warningCountBefore) {
warnings.push(
`The API Report for ${packageDir} introduces new warnings. ` +
'Please fix these warnings in order to keep the API Reports tidy.',
);
}
}
if (remainingReportFiles.size > 0) {
if (isLocalBuild) {
for (const f of remainingReportFiles) {
fs.rmSync(resolvePath(projectFolder, f));
console.log(`Deleted deprecated API report ${f}`);
}
} else {
const staleList = [...remainingReportFiles]
.map(f => join(packageDir, f))
.join(', ');
throw new Error(
`The API Report(s) ${staleList} are no longer relevant and should be deleted`,
);
}
}
}
if (warnings.length > 0) {
console.warn();
for (const warning of warnings) {
console.warn(warning);
}
console.warn();
}
}