Merge branch 'master' into feature/techdocs-e2e

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
2021-09-28 11:12:31 +02:00
parent 5b31a86950 322b60d5f0
commit 174c574311
1927 changed files with 64963 additions and 24801 deletions
@@ -31,8 +31,17 @@ import {
  CompilerState,
  ExtractorLogLevel,
 } from '@microsoft/api-extractor';
+import { DocNode, IDocNodeContainerParameters } from '@microsoft/tsdoc';
 import { ApiPackage, ApiModel } from '@microsoft/api-extractor-model';
-import { MarkdownDocumenter } from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter';
+import {
+  IMarkdownDocumenterOptions,
+  MarkdownDocumenter,
+} from '@microsoft/api-documenter/lib/documenters/MarkdownDocumenter';
+import { DocTable } from '@microsoft/api-documenter/lib/nodes/DocTable';
+import { DocTableRow } from '@microsoft/api-documenter/lib/nodes/DocTableRow';
+import { DocHeading } from '@microsoft/api-documenter/lib/nodes/DocHeading';
+import { CustomMarkdownEmitter } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter';
+import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter';

 const tmpDir = resolvePath(__dirname, '../node_modules/.cache/api-extractor');

@@ -50,17 +59,16 @@ const {
 } = require('@rushstack/node-core-library/lib/PackageJsonLookup');

 const old = PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor;
-PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = function tryGetPackageJsonFilePathForPatch(
-  path: string,
-) {
-  if (
-    path.includes('@material-ui') &&
-    !dirname(path).endsWith('@material-ui')
-  ) {
-    return undefined;
-  }
-  return old.call(this, path);
-};
+PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor =
+  function tryGetPackageJsonFilePathForPatch(path: string) {
+    if (
+      path.includes('@material-ui') &&
+      !dirname(path).endsWith('@material-ui')
+    ) {
+      return undefined;
+    }
+    return old.call(this, path);
+  };

 /**
 * Another monkey patch where we apply prettier to the API reports. This has to be patched into
@@ -73,28 +81,26 @@ const {

 const originalGenerateReviewFileContent =
  ApiReportGenerator.generateReviewFileContent;
-ApiReportGenerator.generateReviewFileContent = function decoratedGenerateReviewFileContent(
-  ...args
-) {
-  const content = originalGenerateReviewFileContent.apply(this, args);
-  return prettier.format(content, {
-    ...require('@spotify/prettier-config'),
-    parser: 'markdown',
-  });
-};
+ApiReportGenerator.generateReviewFileContent =
+  function decoratedGenerateReviewFileContent(...args) {
+    const content = originalGenerateReviewFileContent.apply(this, args);
+    return prettier.format(content, {
+      ...require('@spotify/prettier-config'),
+      parser: 'markdown',
+    });
+  };

 const PACKAGE_ROOTS = ['packages', 'plugins'];

 const SKIPPED_PACKAGES = [
-  'packages/app',
-  'packages/backend',
-  'packages/cli',
-  'packages/codemods',
-  'packages/create-app',
-  'packages/docgen',
-  'packages/e2e-test',
-  'packages/storybook',
-  'packages/techdocs-cli',
+  join('packages', 'app'),
+  join('packages', 'backend'),
+  join('packages', 'cli'),
+  join('packages', 'codemods'),
+  join('packages', 'create-app'),
+  join('packages', 'e2e-test'),
+  join('packages', 'storybook'),
+  join('packages', 'techdocs-cli'),
 ];

 async function findPackageDirs() {
@@ -104,7 +110,7 @@ async function findPackageDirs() {
  for (const packageRoot of PACKAGE_ROOTS) {
    const dirs = await fs.readdir(resolvePath(projectRoot, packageRoot));
    for (const dir of dirs) {
-      const fullPackageDir = resolvePath(packageRoot, dir);
+      const fullPackageDir = resolvePath(projectRoot, packageRoot, dir);

      const stat = await fs.stat(fullPackageDir);
      if (!stat.isDirectory()) {
@@ -218,9 +224,11 @@ async function runApiExtraction({
    // The `packageFolder` needs to point to the location within `dist-types` in order for relative
    // paths to be logged. Unfortunately the `prepare` method above derives it from the `packageJsonFullPath`,
    // which needs to point to the actual file, so we override `packageFolder` afterwards.
-    (extractorConfig as {
-      packageFolder: string;
-    }).packageFolder = packageFolder;
+    (
+      extractorConfig as {
+        packageFolder: string;
+      }
+    ).packageFolder = packageFolder;

    if (!compilerState) {
      compilerState = CompilerState.create(extractorConfig, {
@@ -298,10 +306,13 @@ async function runApiExtraction({
  }
 }

-function isComponentMember(member: any) {
-  // React components are annotated with @component, and we want to skip those
-  return Boolean(member.docComment.match(/\n\s*\**\s*@component/m));
-}
+/*
+WARNING: Bring a blanket if you're gonna read the code below
+
+There's some weird shit going on here, and it's because we cba
+forking rushstash to modify the api-documenter markdown generation,
+which otherwise is the recommended way to do customizations.
+*/

 async function buildDocs({
  inputDir,
@@ -310,6 +321,8 @@ async function buildDocs({
  inputDir: string;
  outputDir: string;
 }) {
+  // We start by constructing our own model from the files so that
+  // we get a change to modify them, as the model is otherwise read-only.
  const parseFile = async (filename: string): Promise<any> => {
    console.log(`Reading ${filename}`);
    return fs.readJson(resolvePath(inputDir, filename));
@@ -324,9 +337,7 @@ async function buildDocs({

  const newModel = new ApiModel();
  for (const serialized of serializedPackages) {
-    serialized.members[0].members = serialized.members[0].members.filter(
-      member => !isComponentMember(member),
-    );
+    // Add any docs filtering logic here

    const pkg = ApiPackage.deserialize(
      serialized,
@@ -335,10 +346,181 @@ async function buildDocs({
    newModel.addMember(pkg);
  }

-  await fs.remove(outputDir);
-  await fs.ensureDir(outputDir);
+  // The doc AST need to be extended with custom nodes if we want to
+  // add any extra content.
+  // This one is for the YAML front matter that we need for the microsite.
+  class DocFrontMatter extends DocNode {
+    static kind = 'DocFrontMatter';

-  const documenter = new MarkdownDocumenter({
+    public readonly values: { [name: string]: unknown };
+
+    public constructor(
+      parameters: IDocNodeContainerParameters & {
+        values: { [name: string]: unknown };
+      },
+    ) {
+      super(parameters);
+      this.values = parameters.values;
+    }
+
+    /** @override */
+    public get kind(): string {
+      return DocFrontMatter.kind;
+    }
+  }
+
+  // This is where we actually write the markdown and where we can hook
+  // in the rendering of our own nodes.
+  class CustomCustomMarkdownEmitter extends CustomMarkdownEmitter {
+    /** @override */
+    protected writeNode(
+      docNode: DocNode,
+      context: IMarkdownEmitterContext,
+      docNodeSiblings: boolean,
+    ): void {
+      switch (docNode.kind) {
+        case DocFrontMatter.kind: {
+          const node = docNode as DocFrontMatter;
+          context.writer.writeLine('---');
+          for (const [name, value] of Object.entries(node.values)) {
+            if (value) {
+              context.writer.writeLine(`${name}: ${value}`);
+            }
+          }
+          context.writer.writeLine('---');
+          context.writer.writeLine();
+          break;
+        }
+        default:
+          super.writeNode(docNode, context, docNodeSiblings);
+      }
+    }
+
+    /** @override */
+    emit(stringBuilder, docNode, options) {
+      // Hack to get rid of the leading comment of each file, since
+      // we want the front matter to come first
+      stringBuilder._chunks.length = 0;
+      return super.emit(stringBuilder, docNode, options);
+    }
+  }
+
+  class CustomMarkdownDocumenter extends (MarkdownDocumenter as any) {
+    constructor(options: IMarkdownDocumenterOptions) {
+      super(options);
+
+      // It's a strict model, we gotta register the allowed usage of our new node
+      this._tsdocConfiguration.docNodeManager.registerDocNodes(
+        '@backstage/docs',
+        [{ docNodeKind: DocFrontMatter.kind, constructor: DocFrontMatter }],
+      );
+      this._tsdocConfiguration.docNodeManager.registerAllowableChildren(
+        'Paragraph',
+        [DocFrontMatter.kind],
+      );
+
+      this._markdownEmitter = new CustomCustomMarkdownEmitter(newModel);
+    }
+
+    // We don't really get many chances to modify the generated AST
+    // so we hook in wherever we can. In this case we add the front matter
+    // just before writing the breadcrumbs at the top.
+    /** @override */
+    _writeBreadcrumb(output, apiItem) {
+      let title;
+      let description;
+
+      const name = apiItem.getScopedNameWithinPackage();
+      if (name) {
+        title = name;
+        description = `API reference for ${apiItem.getScopedNameWithinPackage()}`;
+      } else if (apiItem.kind === 'Model') {
+        title = 'Package Index';
+        description = 'Index of all Backstage Packages';
+      } else {
+        title = apiItem.name;
+        description = `API Reference for ${apiItem.name}`;
+      }
+
+      // Add our front matter
+      output.appendNodeInParagraph(
+        new DocFrontMatter({
+          configuration: this._tsdocConfiguration,
+          values: {
+            id: this._getFilenameForApiItem(apiItem).slice(0, -3),
+            title,
+            description,
+          },
+        }),
+      );
+
+      // Now write the actual breadcrumbs
+      super._writeBreadcrumb(output, apiItem);
+
+      // We wanna ignore the header that always gets written after the breadcrumb
+      // This otherwise becomes more or less a duplicate of the title in the front matter
+      const oldAppendNode = output.appendNode;
+      output.appendNode = () => {
+        output.appendNode = oldAppendNode;
+      };
+    }
+
+    _writeModelTable(output, apiModel): void {
+      const configuration = this._tsdocConfiguration;
+
+      const packagesTable = new DocTable({
+        configuration,
+        headerTitles: ['Package', 'Description'],
+      });
+
+      const pluginsTable = new DocTable({
+        configuration,
+        headerTitles: ['Package', 'Description'],
+      });
+
+      for (const apiMember of apiModel.members) {
+        const row = new DocTableRow({ configuration }, [
+          this._createTitleCell(apiMember),
+          this._createDescriptionCell(apiMember),
+        ]);
+
+        if (apiMember.kind === 'Package') {
+          this._writeApiItemPage(apiMember);
+
+          if (apiMember.name.startsWith('@backstage/plugin-')) {
+            pluginsTable.addRow(row);
+          } else {
+            packagesTable.addRow(row);
+          }
+        }
+      }
+
+      if (packagesTable.rows.length > 0) {
+        output.appendNode(
+          new DocHeading({
+            configuration: this._tsdocConfiguration,
+            title: 'Packages',
+          }),
+        );
+        output.appendNode(packagesTable);
+      }
+
+      if (pluginsTable.rows.length > 0) {
+        output.appendNode(
+          new DocHeading({
+            configuration: this._tsdocConfiguration,
+            title: 'Plugins',
+          }),
+        );
+        output.appendNode(pluginsTable);
+      }
+    }
+  }
+
+  // This is root of the documentation generation, but it's not directly
+  // responsible for generating markdown, it just constructs an AST that
+  // is the consumed by an emitter to actually write the files.
+  const documenter = new CustomMarkdownDocumenter({
    apiModel: newModel,
    documenterConfig: {
      outputTarget: 'markdown',
@@ -350,6 +532,9 @@ async function buildDocs({
    outputFolder: outputDir,
  });

+  // Clean up existing stuff and write ALL the docs!
+  await fs.remove(outputDir);
+  await fs.ensureDir(outputDir);
  documenter.generateFiles();
 }

@@ -139,9 +139,8 @@ async function findExternalDocsLinks(dir) {
    const match = content.match(/---(?:\r|\n|.)*^id: (.*)$/m);

    // Both docs with an id and without should remove trailing /index
-    const realPath = (match
-      ? joinPath(dirname(url), match[1])
-      : url.replace(/\.md$/, '')
+    const realPath = (
+      match ? joinPath(dirname(url), match[1]) : url.replace(/\.md$/, '')
    ).replace(/\/index$/, '');

    paths.set(url, realPath);