From 52da6addc398e2c95804bdd3b44435447c8a4d79 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 13 Apr 2025 19:53:00 -0400 Subject: [PATCH] add caching to the build to speed up incremental changes Signed-off-by: aramissennyeydd --- .github/workflows/deploy_microsite.yml | 36 +++++ .github/workflows/verify_microsite.yml | 36 +++++ packages/repo-tools/package.json | 1 + .../src/commands/package-docs/Cache.ts | 140 ++++++++++++++++++ .../src/commands/package-docs/command.ts | 97 +++++++----- yarn.lock | 1 + 6 files changed, 276 insertions(+), 35 deletions(-) create mode 100644 packages/repo-tools/src/commands/package-docs/Cache.ts diff --git a/.github/workflows/deploy_microsite.yml b/.github/workflows/deploy_microsite.yml index 6e59381f46..1ed50be22f 100644 --- a/.github/workflows/deploy_microsite.yml +++ b/.github/workflows/deploy_microsite.yml @@ -80,8 +80,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -91,6 +101,14 @@ jobs: if-no-files-found: error retention-days: 1 + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -147,8 +165,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -158,6 +186,14 @@ jobs: if-no-files-found: error retention-days: 1 + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }} + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable diff --git a/.github/workflows/verify_microsite.yml b/.github/workflows/verify_microsite.yml index 8142592aaa..674ecb23b2 100644 --- a/.github/workflows/verify_microsite.yml +++ b/.github/workflows/verify_microsite.yml @@ -85,8 +85,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -94,6 +104,14 @@ jobs: name: stable-reference-beta path: type-docs/ + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -149,8 +167,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -158,6 +186,14 @@ jobs: name: next-reference-beta path: type-docs/ + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }} + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 154cb015aa..1963a56f45 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -69,6 +69,7 @@ "commander": "^12.0.0", "fs-extra": "^11.2.0", "glob": "^8.0.3", + "globby": "^11.0.0", "is-glob": "^4.0.3", "js-yaml": "^4.1.0", "just-diff": "^6.0.2", diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts new file mode 100644 index 0000000000..710997a0a9 --- /dev/null +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { readFile, writeFile, cp } from 'fs/promises'; +import globby from 'globby'; +import workerPath, { dirname } from 'path'; +import crypto from 'crypto'; +import { Lockfile } from '@backstage/cli-node'; +import { paths as cliPaths } from '../../lib/paths'; +import { mkdirp } from 'fs-extra'; + +const version = '1'; +const CACHE_FILE = 'cache.json'; + +interface CacheEntry { + hash: string; + packageName: string; + restoreTo: string; + version: string; +} + +export class PackageDocsCache { + private keyCache: Map; + constructor( + private readonly lockfile: Lockfile, + private readonly cache: Map, + private readonly cacheDir: string, + ) { + this.keyCache = new Map(); + } + static async loadAsync(cacheDir: string, lockfile: Lockfile) { + const cacheFiles = await globby(`**/${CACHE_FILE}`, { + cwd: cacheDir, + }); + const map = new Map(); + for (const file of cacheFiles) { + console.log(file); + const pkg = dirname(file); + const cache = await readFile(workerPath.join(cacheDir, file), 'utf-8'); + const cacheJson = JSON.parse(cache); + map.set(pkg, cacheJson); + } + return new PackageDocsCache(lockfile, map, cacheDir); + } + + async directoryToName(directory: string) { + const packageJson = await readFile( + workerPath.join(directory, 'package.json'), + 'utf-8', + ); + return JSON.parse(packageJson).name; + } + + async toKey(pkg: string) { + if (this.keyCache.has(pkg)) { + return this.keyCache.get(pkg)!; + } + const name = await this.directoryToName(pkg); + const result = await globby('src/**', { + gitignore: true, + onlyFiles: true, + cwd: pkg, + }); + + const hash = crypto.createHash('sha1'); + hash.update(version); + hash.update('\0'); + + for (const path of result.sort()) { + const absPath = cliPaths.resolveTargetRoot(pkg, path); + const pathInPackage = workerPath.join(absPath, path); + hash.update(pathInPackage); + hash.update('\0'); + hash.update(await readFile(absPath)); + hash.update('\0'); + } + hash.update(this.lockfile.getDependencyTreeHash(name)); + hash.update('\0'); + const hashString = hash.digest('hex'); + this.keyCache.set(pkg, hashString); + return hashString; + } + + async has(pkg: string) { + const cache = this.cache.get(pkg); + if (!cache) { + return false; + } + const hashString = await this.toKey(pkg); + return cache.hash === hashString; + } + + async restore(pkg: string) { + if (!this.has(pkg)) { + throw new Error(`Cache entry for ${pkg} not found`); + } + const cacheEntry = this.cache.get(pkg); + const restoreTo = cacheEntry!.restoreTo; + const cacheDir = workerPath.join(this.cacheDir, pkg); + const contentsDir = workerPath.join(cacheDir, 'contents'); + + const targetDir = cliPaths.resolveTargetRoot(restoreTo); + await mkdirp(targetDir); + console.log(contentsDir, targetDir); + await cp(contentsDir, targetDir, { recursive: true }); + } + + async write(pkg: string, contentDirectory: string) { + const cacheDir = workerPath.join(this.cacheDir, pkg); + const contentsDir = workerPath.join(cacheDir, 'contents'); + await mkdirp(cacheDir); + const hashString = await this.toKey(pkg); + await cp(contentDirectory, contentsDir, { recursive: true }); + const cacheEntry: CacheEntry = { + hash: hashString, + packageName: await this.directoryToName(pkg), + restoreTo: workerPath.relative( + cliPaths.resolveTargetRoot(), + contentDirectory, + ), + version, + }; + await writeFile( + workerPath.join(cacheDir, CACHE_FILE), + JSON.stringify(cacheEntry), + ); + } +} diff --git a/packages/repo-tools/src/commands/package-docs/command.ts b/packages/repo-tools/src/commands/package-docs/command.ts index 8c30b06145..50f52c6b87 100644 --- a/packages/repo-tools/src/commands/package-docs/command.ts +++ b/packages/repo-tools/src/commands/package-docs/command.ts @@ -17,8 +17,11 @@ import { exec } from 'child_process'; import { promisify } from 'util'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { createTemporaryTsConfig } from './utils'; -import { mkdir, readFile, writeFile } from 'fs/promises'; +import { readFile, writeFile } from 'fs/promises'; import pLimit from 'p-limit'; +import { mkdirp } from 'fs-extra'; +import { PackageDocsCache } from './Cache'; +import { Lockfile } from '@backstage/cli-node'; const limit = pLimit(8); @@ -38,6 +41,7 @@ const EXCLUDE = [ 'packages/techdocs-cli-embedded-app', 'packages/yarn-plugin', 'packages/backend', + 'packages/backend-legacy', ]; const HIGHLIGHT_LANGUAGES = [ @@ -54,6 +58,8 @@ const HIGHLIGHT_LANGUAGES = [ 'json', ]; +const CACHE_DIR = '.cache/package-docs'; + function getExports(packageJson: any) { if (packageJson.exports) { return Object.values(packageJson.exports).filter( @@ -75,41 +81,35 @@ async function generateDocJson(pkg: string) { !exports.length || !exports.some(e => e.startsWith('src') || e.startsWith('./src')) ) { - return; + return false; } - try { - await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), { - recursive: true, - }); + await mkdirp(cliPaths.resolveTargetRoot(`dist-types`, pkg)); - const { stdout, stderr } = await execAsync( - [ - cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), - '--json', - cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'), - '--tsconfig', - temporaryTsConfigPath, - '--basePath', - cliPaths.targetRoot, - '--skipErrorChecking', - ...(getExports(packageJson).flatMap(e => [ - '--entryPoints', - e, - ]) as string[]), - ].join(' '), - { - cwd: pkg, - env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, - }, - ); - console.log(`### Processed ${pkg}`); - console.log(stdout); - console.error(stderr); - } catch (e) { - console.error('Failed to generate docs for', pkg); - console.error(e); - } + const { stdout, stderr } = await execAsync( + [ + cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), + '--json', + cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'), + '--tsconfig', + temporaryTsConfigPath, + '--basePath', + cliPaths.targetRoot, + '--skipErrorChecking', + ...(getExports(packageJson).flatMap(e => [ + '--entryPoints', + e, + ]) as string[]), + ].join(' '), + { + cwd: pkg, + env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, + }, + ); + console.log(`### Processed ${pkg}`); + console.log(stdout); + console.error(stderr); + return true; } export default async function packageDocs(paths: string[] = [], opts: any) { @@ -120,6 +120,12 @@ export default async function packageDocs(paths: string[] = [], opts: any) { exclude: opts.exclude, }); + await mkdirp(CACHE_DIR); + const cache = await PackageDocsCache.loadAsync( + CACHE_DIR, + await Lockfile.load(cliPaths.resolveTargetRoot('yarn.lock')), + ); + console.log(`### Generating docs.`); await Promise.all( selectedPackageDirs.map(pkg => @@ -127,8 +133,29 @@ export default async function packageDocs(paths: string[] = [], opts: any) { if (EXCLUDE.includes(pkg)) { return; } - console.log(`### Processing ${pkg}`); - await generateDocJson(pkg); + if (await cache.has(pkg)) { + console.log(`### Skipping ${pkg} due to cache hit`); + try { + await cache.restore(pkg); + return; + } catch (e) { + console.error('Failed to restore cache for', pkg); + console.error(e); + } + } + try { + console.log(`### Processing ${pkg}`); + const success = await generateDocJson(pkg); + if (success) { + await cache.write( + pkg, + cliPaths.resolveTargetRoot(`dist-types`, pkg), + ); + } + } catch (e) { + console.error('Failed to generate docs for', pkg); + console.error(e); + } }), ), ); diff --git a/yarn.lock b/yarn.lock index 20037a1652..7c9d7d2c58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8414,6 +8414,7 @@ __metadata: commander: "npm:^12.0.0" fs-extra: "npm:^11.2.0" glob: "npm:^8.0.3" + globby: "npm:^11.0.0" is-glob: "npm:^4.0.3" js-yaml: "npm:^4.1.0" just-diff: "npm:^6.0.2"