From 7f089a8649fbde09cccc883a5db5aba758b3827f Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 9 Feb 2025 13:00:42 -0500 Subject: [PATCH] update to generate + merge as separate steps, runtime down to 40s Signed-off-by: aramissennyeydd --- .gitignore | 5 + .../src/commands/package-docs/command.ts | 188 +++++++++++++++--- .../src/commands/package-docs/utils.ts | 44 ++++ typedoc.json | 26 --- 4 files changed, 205 insertions(+), 58 deletions(-) create mode 100644 packages/repo-tools/src/commands/package-docs/utils.ts delete mode 100644 typedoc.json diff --git a/.gitignore b/.gitignore index 865918c1f7..6627583fce 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,8 @@ knip.json # Schemathesis temporary files .hypothesis/ .cassettes/ + +# Typedocs temporary files +type-docs +docs.json +tsconfig.typedoc.tmp.json \ No newline at end of file diff --git a/packages/repo-tools/src/commands/package-docs/command.ts b/packages/repo-tools/src/commands/package-docs/command.ts index 3fc902ab62..b9ebfec85c 100644 --- a/packages/repo-tools/src/commands/package-docs/command.ts +++ b/packages/repo-tools/src/commands/package-docs/command.ts @@ -13,42 +13,166 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { spawn } from 'child_process'; +import { exec } from 'child_process'; import { promisify } from 'util'; -import { paths } from '../../lib/paths'; +import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; +import { createTemporaryTsConfig } from './utils'; +import { mkdir, readFile, writeFile } from 'fs/promises'; +import pLimit from 'p-limit'; -const execAsync = promisify(spawn); +const limit = pLimit(8); -export default async function packageDocs() { - // const packages = await PackageGraph.listTargetPackages(); - // for (const pkg of packages) { - // console.log(path.relative(paths.targetRoot, pkg.dir)); - // const configPath = path.join(pkg.dir, 'typedoc.json'); - // try { - // const DEFAULT_CONFIG = { - // extends: ['../../typedoc.base.jsonc'], - // entryPoints: - // Object.values(pkg.packageJson.exports ?? {}) ?? pkg.packageJson.main, - // }; - // await writeFile(configPath, JSON.stringify(DEFAULT_CONFIG, null, 2)); - // } catch (e) { - // console.error(`Failed to generate docs for ${pkg.packageJson.name}`); - // console.error(e); - // } finally { - // } - // } - console.log(`Generating docs.`); - await execAsync( - paths.resolveTargetRoot('node_modules/.bin/typedoc'), - ['--out', 'type-docs', '--entryPointStrategy', 'packages'], +const execAsync = promisify(exec); + +const EXCLUDE = [ + 'packages/app', + 'packages/app-next', + 'packages/app-next-example-plugin', + 'packages/cli', + 'packages/cli-common', + 'packages/cli-node', + 'packages/e2e-test', + 'packages/e2e-test-utils', + 'packages/opaque-internal', + 'packages/techdocs-cli', + 'packages/techdocs-cli-embedded-app', + 'packages/yarn-plugin', + 'packages/backend', +]; + +const HIGHLIGHT_LANGUAGES = [ + 'ts', + 'tsx', + 'yaml', + 'bash', + 'sh', + 'shell', + 'yml', + 'jsx', + 'diff', + 'js', + 'json', +]; + +function getExports(packageJson: any) { + if (packageJson.exports) { + return Object.values(packageJson.exports).filter( + (e: any) => !(e as string).endsWith('package.json'), + ); + } + return [packageJson.main]; +} + +async function generateDocJson(pkg: string) { + const temporaryTsConfigPath: string = await createTemporaryTsConfig(pkg); + + const packageJson = JSON.parse( + await readFile(cliPaths.resolveTargetRoot(pkg, 'package.json'), 'utf-8'), + ); + + const exports = getExports(packageJson); + if (!exports.length || !exports.some(e => e.startsWith('src'))) { + return; + } + + try { + await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), { + recursive: true, + }); + + const { stdout, stderr } = await execAsync( + [ + cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), + '--json', + cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'), + '--tsconfig', + temporaryTsConfigPath, + '--basePath', + cliPaths.targetRoot, + '--skipErrorChecking', + ...(getExports(packageJson).flatMap(e => [ + '--entryPoints', + e, + ]) as string[]), + ].join(' '), + { + cwd: pkg, + env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, + }, + ); + console.log(`### Processed ${pkg}`); + console.log(stdout); + console.error(stderr); + } catch (e) { + console.error('Failed to generate docs for', pkg); + console.error(e); + // test + } +} + +export default async function packageDocs(paths: string[] = [], opts: any) { + const selectedPackageDirs = await resolvePackagePaths({ + paths, + include: opts.include, + exclude: opts.exclude, + }); + + console.log(`### Generating docs.`); + await Promise.all( + selectedPackageDirs.map(pkg => + limit(async () => { + if (EXCLUDE.includes(pkg)) { + return; + } + console.log(`### Processing ${pkg}`); + await generateDocJson(pkg); + }), + ), + ); + + const generatedPackageDirs = []; + for (const pkg of selectedPackageDirs) { + try { + const docsJsonPath = cliPaths.resolveTargetRoot( + `dist-types/${pkg}/docs.json`, + ); + const docsJson = JSON.parse(await readFile(docsJsonPath, 'utf-8')); + const index = docsJson.children?.find((child: any) => + child.sources.some((e: any) => e.fileName.endsWith('src/index.ts')), + ); + + if (index) { + index.name = 'index'; + } + await writeFile(docsJsonPath, JSON.stringify(docsJson, null, 2)); + generatedPackageDirs.push(pkg); + } catch (e) { + if (e.code === 'ENOENT') { + console.log('No docs.json found for', pkg); + } else { + throw e; + } + } + } + + const { stdout, stderr } = await execAsync( + [ + cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), + '--entryPointStrategy', + 'merge', + ...generatedPackageDirs.flatMap(pkg => [ + '--entryPoints', + `dist-types/${pkg}/docs.json`, + ]), + ...HIGHLIGHT_LANGUAGES.flatMap(e => ['--highlightLanguages', e]), + '--out', + cliPaths.resolveTargetRoot('type-docs'), + ].join(' '), { - stdio: 'inherit', - cwd: paths.targetRoot, - env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, + cwd: cliPaths.targetRoot, }, ); - // for (const pkg of packages) { - // const configPath = path.join(pkg.dir, 'typedoc.json'); - // await rm(configPath); - // } + + console.log(stdout); + console.error(stderr); } diff --git a/packages/repo-tools/src/commands/package-docs/utils.ts b/packages/repo-tools/src/commands/package-docs/utils.ts new file mode 100644 index 0000000000..2cb95443b6 --- /dev/null +++ b/packages/repo-tools/src/commands/package-docs/utils.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { paths as cliPaths } from '../../lib/paths'; + +export async function createTemporaryTsConfig(dir: string) { + const path = cliPaths.resolveOwnRoot(dir, 'tsconfig.typedoc.tmp.json'); + + process.once('exit', () => { + fs.removeSync(path); + }); + + let assetTypeFile: string[] = []; + + try { + assetTypeFile = [ + require.resolve('@backstage/cli/asset-types/asset-types.d.ts'), + ]; + } catch { + /** ignore */ + } + + await fs.writeJson(path, { + extends: '../../tsconfig.json', + include: [...assetTypeFile, 'src'], + exclude: [], + }); + + return path; +} diff --git a/typedoc.json b/typedoc.json deleted file mode 100644 index e5054a27af..0000000000 --- a/typedoc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "entryPoints": ["packages/*", "plugins/*"], - "exclude": [ - "packages/app", - "packages/app-next", - "packages/app-next-example-plugin", - "packages/cli", - "packages/cli-common", - "packages/cli-node", - "packages/e2e-test", - "packages/e2e-test-utils", - "packages/opaque-internal", - "packages/techdocs-cli", - "packages/techdocs-cli-embedded-app", - "packages/yarn-plugin", - "packages/backend" - ], - "packageOptions": { - "exclude": ["**/package.json"], - "includeVersion": true - }, - "name": "Backstage API References", - "entryPointStrategy": "packages", - "includeVersion": false, - "logLevel": "Verbose" -}