update to generate + merge as separate steps, runtime down to 40s

Signed-off-by: aramissennyeydd <aramis.sennyey@doordash.com>

.gitignore

@@ -174,3 +174,8 @@ knip.json
 # Schemathesis temporary files
 .hypothesis/
 .cassettes/
+
+# Typedocs temporary files
+type-docs
+docs.json
+tsconfig.typedoc.tmp.json

packages/repo-tools/src/commands/package-docs/command.ts

@@ -13,42 +13,166 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { spawn } from 'child_process';
+import { exec } from 'child_process';
 import { promisify } from 'util';
-import { paths } from '../../lib/paths';
+import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
+import { createTemporaryTsConfig } from './utils';
+import { mkdir, readFile, writeFile } from 'fs/promises';
+import pLimit from 'p-limit';
 
-const execAsync = promisify(spawn);
+const limit = pLimit(8);
 
-export default async function packageDocs() {
-  //   const packages = await PackageGraph.listTargetPackages();
-  //   for (const pkg of packages) {
-  //     console.log(path.relative(paths.targetRoot, pkg.dir));
-  //     const configPath = path.join(pkg.dir, 'typedoc.json');
-  //     try {
-  //       const DEFAULT_CONFIG = {
-  //         extends: ['../../typedoc.base.jsonc'],
-  //         entryPoints:
-  //           Object.values(pkg.packageJson.exports ?? {}) ?? pkg.packageJson.main,
-  //       };
-  //       await writeFile(configPath, JSON.stringify(DEFAULT_CONFIG, null, 2));
-  //     } catch (e) {
-  //       console.error(`Failed to generate docs for ${pkg.packageJson.name}`);
-  //       console.error(e);
-  //     } finally {
-  //     }
-  //   }
-  console.log(`Generating docs.`);
-  await execAsync(
-    paths.resolveTargetRoot('node_modules/.bin/typedoc'),
-    ['--out', 'type-docs', '--entryPointStrategy', 'packages'],
+const execAsync = promisify(exec);
+
+const EXCLUDE = [
+  'packages/app',
+  'packages/app-next',
+  'packages/app-next-example-plugin',
+  'packages/cli',
+  'packages/cli-common',
+  'packages/cli-node',
+  'packages/e2e-test',
+  'packages/e2e-test-utils',
+  'packages/opaque-internal',
+  'packages/techdocs-cli',
+  'packages/techdocs-cli-embedded-app',
+  'packages/yarn-plugin',
+  'packages/backend',
+];
+
+const HIGHLIGHT_LANGUAGES = [
+  'ts',
+  'tsx',
+  'yaml',
+  'bash',
+  'sh',
+  'shell',
+  'yml',
+  'jsx',
+  'diff',
+  'js',
+  'json',
+];
+
+function getExports(packageJson: any) {
+  if (packageJson.exports) {
+    return Object.values(packageJson.exports).filter(
+      (e: any) => !(e as string).endsWith('package.json'),
+    );
+  }
+  return [packageJson.main];
+}
+
+async function generateDocJson(pkg: string) {
+  const temporaryTsConfigPath: string = await createTemporaryTsConfig(pkg);
+
+  const packageJson = JSON.parse(
+    await readFile(cliPaths.resolveTargetRoot(pkg, 'package.json'), 'utf-8'),
+  );
+
+  const exports = getExports(packageJson);
+  if (!exports.length || !exports.some(e => e.startsWith('src'))) {
+    return;
+  }
+
+  try {
+    await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), {
+      recursive: true,
+    });
+
+    const { stdout, stderr } = await execAsync(
+      [
+        cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
+        '--json',
+        cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'),
+        '--tsconfig',
+        temporaryTsConfigPath,
+        '--basePath',
+        cliPaths.targetRoot,
+        '--skipErrorChecking',
+        ...(getExports(packageJson).flatMap(e => [
+          '--entryPoints',
+          e,
+        ]) as string[]),
+      ].join(' '),
+      {
+        cwd: pkg,
+        env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
+      },
+    );
+    console.log(`### Processed ${pkg}`);
+    console.log(stdout);
+    console.error(stderr);
+  } catch (e) {
+    console.error('Failed to generate docs for', pkg);
+    console.error(e);
+    // test
+  }
+}
+
+export default async function packageDocs(paths: string[] = [], opts: any) {
+  const selectedPackageDirs = await resolvePackagePaths({
+    paths,
+    include: opts.include,
+    exclude: opts.exclude,
+  });
+
+  console.log(`### Generating docs.`);
+  await Promise.all(
+    selectedPackageDirs.map(pkg =>
+      limit(async () => {
+        if (EXCLUDE.includes(pkg)) {
+          return;
+        }
+        console.log(`### Processing ${pkg}`);
+        await generateDocJson(pkg);
+      }),
+    ),
+  );
+
+  const generatedPackageDirs = [];
+  for (const pkg of selectedPackageDirs) {
+    try {
+      const docsJsonPath = cliPaths.resolveTargetRoot(
+        `dist-types/${pkg}/docs.json`,
+      );
+      const docsJson = JSON.parse(await readFile(docsJsonPath, 'utf-8'));
+      const index = docsJson.children?.find((child: any) =>
+        child.sources.some((e: any) => e.fileName.endsWith('src/index.ts')),
+      );
+
+      if (index) {
+        index.name = 'index';
+      }
+      await writeFile(docsJsonPath, JSON.stringify(docsJson, null, 2));
+      generatedPackageDirs.push(pkg);
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        console.log('No docs.json found for', pkg);
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  const { stdout, stderr } = await execAsync(
+    [
+      cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
+      '--entryPointStrategy',
+      'merge',
+      ...generatedPackageDirs.flatMap(pkg => [
+        '--entryPoints',
+        `dist-types/${pkg}/docs.json`,
+      ]),
+      ...HIGHLIGHT_LANGUAGES.flatMap(e => ['--highlightLanguages', e]),
+      '--out',
+      cliPaths.resolveTargetRoot('type-docs'),
+    ].join(' '),
     {
-      stdio: 'inherit',
-      cwd: paths.targetRoot,
-      env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
+      cwd: cliPaths.targetRoot,
     },
   );
-  //   for (const pkg of packages) {
-  //     const configPath = path.join(pkg.dir, 'typedoc.json');
-  //     await rm(configPath);
-  //   }
+
+  console.log(stdout);
+  console.error(stderr);
 }

packages/repo-tools/src/commands/package-docs/utils.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { paths as cliPaths } from '../../lib/paths';
+
+export async function createTemporaryTsConfig(dir: string) {
+  const path = cliPaths.resolveOwnRoot(dir, 'tsconfig.typedoc.tmp.json');
+
+  process.once('exit', () => {
+    fs.removeSync(path);
+  });
+
+  let assetTypeFile: string[] = [];
+
+  try {
+    assetTypeFile = [
+      require.resolve('@backstage/cli/asset-types/asset-types.d.ts'),
+    ];
+  } catch {
+    /** ignore */
+  }
+
+  await fs.writeJson(path, {
+    extends: '../../tsconfig.json',
+    include: [...assetTypeFile, 'src'],
+    exclude: [],
+  });
+
+  return path;
+}

typedoc.json

@@ -1,26 +0,0 @@
-{
-  "entryPoints": ["packages/*", "plugins/*"],
-  "exclude": [
-    "packages/app",
-    "packages/app-next",
-    "packages/app-next-example-plugin",
-    "packages/cli",
-    "packages/cli-common",
-    "packages/cli-node",
-    "packages/e2e-test",
-    "packages/e2e-test-utils",
-    "packages/opaque-internal",
-    "packages/techdocs-cli",
-    "packages/techdocs-cli-embedded-app",
-    "packages/yarn-plugin",
-    "packages/backend"
-  ],
-  "packageOptions": {
-    "exclude": ["**/package.json"],
-    "includeVersion": true
-  },
-  "name": "Backstage API References",
-  "entryPointStrategy": "packages",
-  "includeVersion": false,
-  "logLevel": "Verbose"
-}