From c01c78b8483225030c6798a80ec63b5c567f2643 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 13 Apr 2025 18:07:40 -0400 Subject: [PATCH 01/10] chore: start building new api reference docs alongside existing build Signed-off-by: aramissennyeydd --- .github/workflows/deploy_microsite.yml | 34 +++++++++++++++++++++++++- .github/workflows/verify_microsite.yml | 30 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_microsite.yml b/.github/workflows/deploy_microsite.yml index ff49b94e56..6e59381f46 100644 --- a/.github/workflows/deploy_microsite.yml +++ b/.github/workflows/deploy_microsite.yml @@ -80,6 +80,17 @@ jobs: if-no-files-found: error retention-days: 1 + - name: build API reference (beta) + run: yarn backstage-repo-tools package-docs + + - name: upload API reference (beta) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: stable-reference-beta + path: type-docs/ + if-no-files-found: error + retention-days: 1 + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -136,6 +147,17 @@ jobs: if-no-files-found: error retention-days: 1 + - name: build API reference (beta) + run: yarn backstage-repo-tools package-docs + + - name: upload API reference (beta) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: next-reference-beta + path: type-docs/ + if-no-files-found: error + retention-days: 1 + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable @@ -265,8 +287,18 @@ jobs: name: storybook path: microsite/build/storybook + - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 + with: + name: stable-reference-beta + path: microsite/build/api/stable/ + + - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 + with: + name: next-reference-beta + path: microsite/build/api/next/ + - name: Check the build output - run: ls microsite/build && ls microsite/build/storybook + run: ls microsite/build && ls microsite/build/storybook && ls microsite/build/api/stable && ls microsite/build/api/next - name: Deploy both microsite and storybook to gh-pages uses: JamesIves/github-pages-deploy-action@62fec3add6773ec5dbbf18d2ee4260911aa35cf4 # v4.6.9 diff --git a/.github/workflows/verify_microsite.yml b/.github/workflows/verify_microsite.yml index ce3be0cf42..8142592aaa 100644 --- a/.github/workflows/verify_microsite.yml +++ b/.github/workflows/verify_microsite.yml @@ -85,6 +85,15 @@ jobs: if-no-files-found: error retention-days: 1 + - name: build API reference (beta) + run: yarn backstage-repo-tools package-docs + + - name: upload API reference (beta) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: stable-reference-beta + path: type-docs/ + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -140,6 +149,15 @@ jobs: if-no-files-found: error retention-days: 1 + - name: build API reference (beta) + run: yarn backstage-repo-tools package-docs + + - name: upload API reference (beta) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: next-reference-beta + path: type-docs/ + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable @@ -271,3 +289,15 @@ jobs: - name: build microsite run: yarn build working-directory: microsite + + - name: download stable reference (beta) + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 + with: + name: stable-reference-beta + path: api/stable/ + + - name: download next reference (beta) + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4 + with: + name: next-reference-beta + path: api/next/ From 52da6addc398e2c95804bdd3b44435447c8a4d79 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 13 Apr 2025 19:53:00 -0400 Subject: [PATCH 02/10] add caching to the build to speed up incremental changes Signed-off-by: aramissennyeydd --- .github/workflows/deploy_microsite.yml | 36 +++++ .github/workflows/verify_microsite.yml | 36 +++++ packages/repo-tools/package.json | 1 + .../src/commands/package-docs/Cache.ts | 140 ++++++++++++++++++ .../src/commands/package-docs/command.ts | 97 +++++++----- yarn.lock | 1 + 6 files changed, 276 insertions(+), 35 deletions(-) create mode 100644 packages/repo-tools/src/commands/package-docs/Cache.ts diff --git a/.github/workflows/deploy_microsite.yml b/.github/workflows/deploy_microsite.yml index 6e59381f46..1ed50be22f 100644 --- a/.github/workflows/deploy_microsite.yml +++ b/.github/workflows/deploy_microsite.yml @@ -80,8 +80,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -91,6 +101,14 @@ jobs: if-no-files-found: error retention-days: 1 + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -147,8 +165,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -158,6 +186,14 @@ jobs: if-no-files-found: error retention-days: 1 + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }} + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable diff --git a/.github/workflows/verify_microsite.yml b/.github/workflows/verify_microsite.yml index 8142592aaa..674ecb23b2 100644 --- a/.github/workflows/verify_microsite.yml +++ b/.github/workflows/verify_microsite.yml @@ -85,8 +85,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -94,6 +104,14 @@ jobs: name: stable-reference-beta path: type-docs/ + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }} + - name: microsite yarn install run: yarn install --immutable working-directory: microsite @@ -149,8 +167,18 @@ jobs: if-no-files-found: error retention-days: 1 + # Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds + - name: restore package-docs cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next- + - name: build API reference (beta) run: yarn backstage-repo-tools package-docs + continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -158,6 +186,14 @@ jobs: name: next-reference-beta path: type-docs/ + # Always save success cache even if there were failures, that way it can be used in re-triggered builds + - name: save package-docs cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + if: always() + with: + path: .cache/package-docs + key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }} + # Also build and upload storybook - name: storybook yarn install run: yarn install --immutable diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 154cb015aa..1963a56f45 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -69,6 +69,7 @@ "commander": "^12.0.0", "fs-extra": "^11.2.0", "glob": "^8.0.3", + "globby": "^11.0.0", "is-glob": "^4.0.3", "js-yaml": "^4.1.0", "just-diff": "^6.0.2", diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts new file mode 100644 index 0000000000..710997a0a9 --- /dev/null +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { readFile, writeFile, cp } from 'fs/promises'; +import globby from 'globby'; +import workerPath, { dirname } from 'path'; +import crypto from 'crypto'; +import { Lockfile } from '@backstage/cli-node'; +import { paths as cliPaths } from '../../lib/paths'; +import { mkdirp } from 'fs-extra'; + +const version = '1'; +const CACHE_FILE = 'cache.json'; + +interface CacheEntry { + hash: string; + packageName: string; + restoreTo: string; + version: string; +} + +export class PackageDocsCache { + private keyCache: Map; + constructor( + private readonly lockfile: Lockfile, + private readonly cache: Map, + private readonly cacheDir: string, + ) { + this.keyCache = new Map(); + } + static async loadAsync(cacheDir: string, lockfile: Lockfile) { + const cacheFiles = await globby(`**/${CACHE_FILE}`, { + cwd: cacheDir, + }); + const map = new Map(); + for (const file of cacheFiles) { + console.log(file); + const pkg = dirname(file); + const cache = await readFile(workerPath.join(cacheDir, file), 'utf-8'); + const cacheJson = JSON.parse(cache); + map.set(pkg, cacheJson); + } + return new PackageDocsCache(lockfile, map, cacheDir); + } + + async directoryToName(directory: string) { + const packageJson = await readFile( + workerPath.join(directory, 'package.json'), + 'utf-8', + ); + return JSON.parse(packageJson).name; + } + + async toKey(pkg: string) { + if (this.keyCache.has(pkg)) { + return this.keyCache.get(pkg)!; + } + const name = await this.directoryToName(pkg); + const result = await globby('src/**', { + gitignore: true, + onlyFiles: true, + cwd: pkg, + }); + + const hash = crypto.createHash('sha1'); + hash.update(version); + hash.update('\0'); + + for (const path of result.sort()) { + const absPath = cliPaths.resolveTargetRoot(pkg, path); + const pathInPackage = workerPath.join(absPath, path); + hash.update(pathInPackage); + hash.update('\0'); + hash.update(await readFile(absPath)); + hash.update('\0'); + } + hash.update(this.lockfile.getDependencyTreeHash(name)); + hash.update('\0'); + const hashString = hash.digest('hex'); + this.keyCache.set(pkg, hashString); + return hashString; + } + + async has(pkg: string) { + const cache = this.cache.get(pkg); + if (!cache) { + return false; + } + const hashString = await this.toKey(pkg); + return cache.hash === hashString; + } + + async restore(pkg: string) { + if (!this.has(pkg)) { + throw new Error(`Cache entry for ${pkg} not found`); + } + const cacheEntry = this.cache.get(pkg); + const restoreTo = cacheEntry!.restoreTo; + const cacheDir = workerPath.join(this.cacheDir, pkg); + const contentsDir = workerPath.join(cacheDir, 'contents'); + + const targetDir = cliPaths.resolveTargetRoot(restoreTo); + await mkdirp(targetDir); + console.log(contentsDir, targetDir); + await cp(contentsDir, targetDir, { recursive: true }); + } + + async write(pkg: string, contentDirectory: string) { + const cacheDir = workerPath.join(this.cacheDir, pkg); + const contentsDir = workerPath.join(cacheDir, 'contents'); + await mkdirp(cacheDir); + const hashString = await this.toKey(pkg); + await cp(contentDirectory, contentsDir, { recursive: true }); + const cacheEntry: CacheEntry = { + hash: hashString, + packageName: await this.directoryToName(pkg), + restoreTo: workerPath.relative( + cliPaths.resolveTargetRoot(), + contentDirectory, + ), + version, + }; + await writeFile( + workerPath.join(cacheDir, CACHE_FILE), + JSON.stringify(cacheEntry), + ); + } +} diff --git a/packages/repo-tools/src/commands/package-docs/command.ts b/packages/repo-tools/src/commands/package-docs/command.ts index 8c30b06145..50f52c6b87 100644 --- a/packages/repo-tools/src/commands/package-docs/command.ts +++ b/packages/repo-tools/src/commands/package-docs/command.ts @@ -17,8 +17,11 @@ import { exec } from 'child_process'; import { promisify } from 'util'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { createTemporaryTsConfig } from './utils'; -import { mkdir, readFile, writeFile } from 'fs/promises'; +import { readFile, writeFile } from 'fs/promises'; import pLimit from 'p-limit'; +import { mkdirp } from 'fs-extra'; +import { PackageDocsCache } from './Cache'; +import { Lockfile } from '@backstage/cli-node'; const limit = pLimit(8); @@ -38,6 +41,7 @@ const EXCLUDE = [ 'packages/techdocs-cli-embedded-app', 'packages/yarn-plugin', 'packages/backend', + 'packages/backend-legacy', ]; const HIGHLIGHT_LANGUAGES = [ @@ -54,6 +58,8 @@ const HIGHLIGHT_LANGUAGES = [ 'json', ]; +const CACHE_DIR = '.cache/package-docs'; + function getExports(packageJson: any) { if (packageJson.exports) { return Object.values(packageJson.exports).filter( @@ -75,41 +81,35 @@ async function generateDocJson(pkg: string) { !exports.length || !exports.some(e => e.startsWith('src') || e.startsWith('./src')) ) { - return; + return false; } - try { - await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), { - recursive: true, - }); + await mkdirp(cliPaths.resolveTargetRoot(`dist-types`, pkg)); - const { stdout, stderr } = await execAsync( - [ - cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), - '--json', - cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'), - '--tsconfig', - temporaryTsConfigPath, - '--basePath', - cliPaths.targetRoot, - '--skipErrorChecking', - ...(getExports(packageJson).flatMap(e => [ - '--entryPoints', - e, - ]) as string[]), - ].join(' '), - { - cwd: pkg, - env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, - }, - ); - console.log(`### Processed ${pkg}`); - console.log(stdout); - console.error(stderr); - } catch (e) { - console.error('Failed to generate docs for', pkg); - console.error(e); - } + const { stdout, stderr } = await execAsync( + [ + cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'), + '--json', + cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'), + '--tsconfig', + temporaryTsConfigPath, + '--basePath', + cliPaths.targetRoot, + '--skipErrorChecking', + ...(getExports(packageJson).flatMap(e => [ + '--entryPoints', + e, + ]) as string[]), + ].join(' '), + { + cwd: pkg, + env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' }, + }, + ); + console.log(`### Processed ${pkg}`); + console.log(stdout); + console.error(stderr); + return true; } export default async function packageDocs(paths: string[] = [], opts: any) { @@ -120,6 +120,12 @@ export default async function packageDocs(paths: string[] = [], opts: any) { exclude: opts.exclude, }); + await mkdirp(CACHE_DIR); + const cache = await PackageDocsCache.loadAsync( + CACHE_DIR, + await Lockfile.load(cliPaths.resolveTargetRoot('yarn.lock')), + ); + console.log(`### Generating docs.`); await Promise.all( selectedPackageDirs.map(pkg => @@ -127,8 +133,29 @@ export default async function packageDocs(paths: string[] = [], opts: any) { if (EXCLUDE.includes(pkg)) { return; } - console.log(`### Processing ${pkg}`); - await generateDocJson(pkg); + if (await cache.has(pkg)) { + console.log(`### Skipping ${pkg} due to cache hit`); + try { + await cache.restore(pkg); + return; + } catch (e) { + console.error('Failed to restore cache for', pkg); + console.error(e); + } + } + try { + console.log(`### Processing ${pkg}`); + const success = await generateDocJson(pkg); + if (success) { + await cache.write( + pkg, + cliPaths.resolveTargetRoot(`dist-types`, pkg), + ); + } + } catch (e) { + console.error('Failed to generate docs for', pkg); + console.error(e); + } }), ), ); diff --git a/yarn.lock b/yarn.lock index 20037a1652..7c9d7d2c58 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8414,6 +8414,7 @@ __metadata: commander: "npm:^12.0.0" fs-extra: "npm:^11.2.0" glob: "npm:^8.0.3" + globby: "npm:^11.0.0" is-glob: "npm:^4.0.3" js-yaml: "npm:^4.1.0" just-diff: "npm:^6.0.2" From d0d1b9933a04dca0b7d005aa0583d0841ab73089 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Mon, 14 Apr 2025 09:29:24 -0400 Subject: [PATCH 03/10] remove continue-on-error stuff Signed-off-by: aramissennyeydd --- .github/workflows/deploy_microsite.yml | 2 -- .github/workflows/verify_microsite.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/deploy_microsite.yml b/.github/workflows/deploy_microsite.yml index 1ed50be22f..a10a5ddd80 100644 --- a/.github/workflows/deploy_microsite.yml +++ b/.github/workflows/deploy_microsite.yml @@ -91,7 +91,6 @@ jobs: - name: build API reference (beta) run: yarn backstage-repo-tools package-docs - continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -176,7 +175,6 @@ jobs: - name: build API reference (beta) run: yarn backstage-repo-tools package-docs - continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/.github/workflows/verify_microsite.yml b/.github/workflows/verify_microsite.yml index 674ecb23b2..e0a6055d74 100644 --- a/.github/workflows/verify_microsite.yml +++ b/.github/workflows/verify_microsite.yml @@ -96,7 +96,6 @@ jobs: - name: build API reference (beta) run: yarn backstage-repo-tools package-docs - continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -178,7 +177,6 @@ jobs: - name: build API reference (beta) run: yarn backstage-repo-tools package-docs - continue-on-error: true - name: upload API reference (beta) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 From bf9a173c9652a1c120d01da9c105d2fe9d96760b Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Wed, 16 Apr 2025 19:23:09 -0400 Subject: [PATCH 04/10] add changeset Signed-off-by: aramissennyeydd --- .changeset/six-mangos-yell.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/six-mangos-yell.md diff --git a/.changeset/six-mangos-yell.md b/.changeset/six-mangos-yell.md new file mode 100644 index 0000000000..65ddae86f8 --- /dev/null +++ b/.changeset/six-mangos-yell.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': minor +--- + +Add support for caching the per-package output from the `package-docs` command. From 7e7ea1477a547c2a57c3a5c876f7b49604a0716d Mon Sep 17 00:00:00 2001 From: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Date: Wed, 7 May 2025 14:38:48 -0400 Subject: [PATCH 05/10] Apply suggestions from code review Signed-off-by: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> --- packages/repo-tools/src/commands/package-docs/Cache.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts index 710997a0a9..4cb0b87288 100644 --- a/packages/repo-tools/src/commands/package-docs/Cache.ts +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -46,7 +46,6 @@ export class PackageDocsCache { }); const map = new Map(); for (const file of cacheFiles) { - console.log(file); const pkg = dirname(file); const cache = await readFile(workerPath.join(cacheDir, file), 'utf-8'); const cacheJson = JSON.parse(cache); @@ -113,7 +112,6 @@ export class PackageDocsCache { const targetDir = cliPaths.resolveTargetRoot(restoreTo); await mkdirp(targetDir); - console.log(contentsDir, targetDir); await cp(contentsDir, targetDir, { recursive: true }); } From 82a46e979d77d8b036fff38215df6084279b0c7c Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Thu, 8 May 2025 11:18:02 -0400 Subject: [PATCH 06/10] start addressing pr feedback Signed-off-by: aramissennyeydd --- packages/repo-tools/package.json | 3 +- .../src/commands/package-docs/Cache.test.ts | 37 +++++++++++++++ .../src/commands/package-docs/Cache.ts | 47 +++++++++++-------- yarn.lock | 7 +-- 4 files changed, 71 insertions(+), 23 deletions(-) create mode 100644 packages/repo-tools/src/commands/package-docs/Cache.test.ts diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 1963a56f45..4c9806dfc6 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -82,7 +82,8 @@ "portfinder": "^1.0.32", "tar": "^6.1.12", "ts-morph": "^24.0.0", - "yaml-diff-patch": "^2.0.0" + "yaml-diff-patch": "^2.0.0", + "zod": "^3.22.4" }, "devDependencies": { "@backstage/backend-test-utils": "workspace:^", diff --git a/packages/repo-tools/src/commands/package-docs/Cache.test.ts b/packages/repo-tools/src/commands/package-docs/Cache.test.ts new file mode 100644 index 0000000000..15205846f3 --- /dev/null +++ b/packages/repo-tools/src/commands/package-docs/Cache.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Lockfile } from '@backstage/cli-node'; +import { PackageDocsCache } from './Cache'; +import { join as joinPath } from 'path'; +import { + createMockDirectory, + MockDirectory, +} from '@backstage/backend-test-utils'; + +describe('PackageDocsCache', () => { + let testDir: MockDirectory; + beforeEach(async () => { + testDir = createMockDirectory(); + }); + it('should be able to parse cache files', async () => { + const cache = await PackageDocsCache.loadAsync( + joinPath(__dirname, 'test-cache'), + new Lockfile(), + ); + cache.add('test', 'test'); + }); +}); diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts index 4cb0b87288..6b64b523e3 100644 --- a/packages/repo-tools/src/commands/package-docs/Cache.ts +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -15,11 +15,12 @@ */ import { readFile, writeFile, cp } from 'fs/promises'; import globby from 'globby'; -import workerPath, { dirname } from 'path'; +import { dirname, join as joinPath, relative } from 'path'; import crypto from 'crypto'; import { Lockfile } from '@backstage/cli-node'; import { paths as cliPaths } from '../../lib/paths'; import { mkdirp } from 'fs-extra'; +import { z } from 'zod'; const version = '1'; const CACHE_FILE = 'cache.json'; @@ -31,10 +32,19 @@ interface CacheEntry { version: string; } +const cacheEntrySchema = z.object({ + hash: z.string(), + packageName: z.string(), + restoreTo: z.string(), + version: z.string(), +}); + export class PackageDocsCache { + // A map of package directory to package hash. private keyCache: Map; constructor( private readonly lockfile: Lockfile, + // A map of package directory to cache entry. private readonly cache: Map, private readonly cacheDir: string, ) { @@ -47,16 +57,21 @@ export class PackageDocsCache { const map = new Map(); for (const file of cacheFiles) { const pkg = dirname(file); - const cache = await readFile(workerPath.join(cacheDir, file), 'utf-8'); - const cacheJson = JSON.parse(cache); - map.set(pkg, cacheJson); + const cache = await readFile(joinPath(cacheDir, file), 'utf-8'); + try { + const cacheJson = JSON.parse(cache); + const parsed = cacheEntrySchema.parse(cacheJson); + map.set(pkg, parsed); + } catch (e) { + console.error(`Skipping unparseable cache file ${file}: ${e}`); + } } return new PackageDocsCache(lockfile, map, cacheDir); } async directoryToName(directory: string) { const packageJson = await readFile( - workerPath.join(directory, 'package.json'), + joinPath(directory, 'package.json'), 'utf-8', ); return JSON.parse(packageJson).name; @@ -79,7 +94,7 @@ export class PackageDocsCache { for (const path of result.sort()) { const absPath = cliPaths.resolveTargetRoot(pkg, path); - const pathInPackage = workerPath.join(absPath, path); + const pathInPackage = joinPath(absPath, path); hash.update(pathInPackage); hash.update('\0'); hash.update(await readFile(absPath)); @@ -107,8 +122,8 @@ export class PackageDocsCache { } const cacheEntry = this.cache.get(pkg); const restoreTo = cacheEntry!.restoreTo; - const cacheDir = workerPath.join(this.cacheDir, pkg); - const contentsDir = workerPath.join(cacheDir, 'contents'); + const cacheDir = joinPath(this.cacheDir, pkg); + const contentsDir = joinPath(cacheDir, 'contents'); const targetDir = cliPaths.resolveTargetRoot(restoreTo); await mkdirp(targetDir); @@ -116,23 +131,17 @@ export class PackageDocsCache { } async write(pkg: string, contentDirectory: string) { - const cacheDir = workerPath.join(this.cacheDir, pkg); - const contentsDir = workerPath.join(cacheDir, 'contents'); - await mkdirp(cacheDir); + const cacheDir = joinPath(this.cacheDir, pkg); + const contentsDir = joinPath(cacheDir, 'contents'); + await mkdirp(contentsDir); const hashString = await this.toKey(pkg); await cp(contentDirectory, contentsDir, { recursive: true }); const cacheEntry: CacheEntry = { hash: hashString, packageName: await this.directoryToName(pkg), - restoreTo: workerPath.relative( - cliPaths.resolveTargetRoot(), - contentDirectory, - ), + restoreTo: relative(cliPaths.resolveTargetRoot(), contentDirectory), version, }; - await writeFile( - workerPath.join(cacheDir, CACHE_FILE), - JSON.stringify(cacheEntry), - ); + await writeFile(joinPath(cacheDir, CACHE_FILE), JSON.stringify(cacheEntry)); } } diff --git a/yarn.lock b/yarn.lock index 7c9d7d2c58..254464caa9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8429,6 +8429,7 @@ __metadata: ts-morph: "npm:^24.0.0" typedoc: "npm:^0.27.6" yaml-diff-patch: "npm:^2.0.0" + zod: "npm:^3.22.4" peerDependencies: "@microsoft/api-extractor-model": "*" "@microsoft/tsdoc": "*" @@ -48020,9 +48021,9 @@ __metadata: linkType: hard "zod@npm:^3.22.4": - version: 3.23.8 - resolution: "zod@npm:3.23.8" - checksum: 10/846fd73e1af0def79c19d510ea9e4a795544a67d5b34b7e1c4d0425bf6bfd1c719446d94cdfa1721c1987d891321d61f779e8236fde517dc0e524aa851a6eff1 + version: 3.24.4 + resolution: "zod@npm:3.24.4" + checksum: 10/3d545792fa54bb27ee5dbc34a5709e81f603185fcc94c8204b5d95c20dc4c81d870ff9c51f3884a30ef05cdc601449f4c4df254ac4783f0827b1faed7c1cdb48 languageName: node linkType: hard From 1bab3f6e6f8a9f852cb0838d463f427d5c31a562 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Thu, 8 May 2025 21:40:41 -0400 Subject: [PATCH 07/10] add test case and update usage Signed-off-by: aramissennyeydd --- .../src/commands/package-docs/Cache.test.ts | 153 +++++++++++++++++- .../src/commands/package-docs/Cache.ts | 31 ++-- .../src/commands/package-docs/command.ts | 5 +- .../src/commands/package-docs/constants.ts | 18 +++ 4 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 packages/repo-tools/src/commands/package-docs/constants.ts diff --git a/packages/repo-tools/src/commands/package-docs/Cache.test.ts b/packages/repo-tools/src/commands/package-docs/Cache.test.ts index 15205846f3..3fe52dcabd 100644 --- a/packages/repo-tools/src/commands/package-docs/Cache.test.ts +++ b/packages/repo-tools/src/commands/package-docs/Cache.test.ts @@ -16,22 +16,163 @@ import { Lockfile } from '@backstage/cli-node'; import { PackageDocsCache } from './Cache'; -import { join as joinPath } from 'path'; import { createMockDirectory, MockDirectory, } from '@backstage/backend-test-utils'; +import { readFile } from 'fs/promises'; +import { join as joinPath } from 'path'; + +jest.mock('crypto', () => { + const hash = { + update: jest.fn(), + digest: jest.fn().mockReturnValue('test'), + }; + return { + createHash: jest.fn().mockReturnValue(hash), + }; +}); describe('PackageDocsCache', () => { let testDir: MockDirectory; - beforeEach(async () => { + beforeAll(async () => { testDir = createMockDirectory(); }); + afterEach(async () => { + testDir.clear(); + }); it('should be able to parse cache files', async () => { - const cache = await PackageDocsCache.loadAsync( - joinPath(__dirname, 'test-cache'), - new Lockfile(), + testDir.addContent({ + '.cache': { + 'package-docs': { + test: { + 'cache.json': JSON.stringify({ + hash: 'test', + packageName: 'test', + restoreTo: 'test', + version: '1', + }), + }, + }, + }, + test: { + 'package.json': JSON.stringify({ + name: '@test/test', + }), + }, + }); + const lockfile = { + getDependencyTreeHash: () => 'test', + } as any as Lockfile; + const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile); + expect(await cache.has('test')).toBe(true); + }); + + it('should be able to restore cache', async () => { + testDir.addContent({ + '.cache': { + 'package-docs': { + test: { + 'cache.json': JSON.stringify({ + hash: 'test', + packageName: 'test', + restoreTo: 'test', + version: '1', + }), + contents: { + 'src/index.ts': 'export const test = "test";', + }, + }, + }, + }, + test: { + 'package.json': JSON.stringify({ + name: '@test/test', + }), + }, + }); + const lockfile = { + getDependencyTreeHash: () => 'test', + } as any as Lockfile; + const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile); + await cache.restore('test'); + expect( + await readFile(joinPath(testDir.path, 'test', 'src/index.ts'), 'utf-8'), + ).toBe('export const test = "test";'); + }); + + it('should be able to write cache', async () => { + testDir.addContent({ + '.cache': {}, + test: { + 'package.json': JSON.stringify({ + name: '@test/test', + }), + 'src/index.ts': 'export const test = "test";', + }, + }); + const lockfile = { + getDependencyTreeHash: () => 'test', + } as any as Lockfile; + const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile); + await cache.write('test', joinPath(testDir.path, 'test')); + expect( + await readFile( + joinPath(testDir.path, '.cache', 'package-docs', 'test', 'cache.json'), + 'utf-8', + ), + ).toBe( + JSON.stringify({ + hash: 'test', + packageName: '@test/test', + restoreTo: 'test', + version: '1', + }), ); - cache.add('test', 'test'); + expect( + await readFile( + joinPath( + testDir.path, + '.cache', + 'package-docs', + 'test', + 'contents', + 'src/index.ts', + ), + 'utf-8', + ), + ).toBe('export const test = "test";'); + }); + + it.each([ + { + content: JSON.stringify({ + hash: 'test', + packageName: 'test', + restoreTo: 'test', + version: '2', + }), + }, + { + content: JSON.stringify({ + hash: 'test', + packageName: 1, + }), + }, + ])('should skip invalid cache files', async content => { + testDir.addContent({ + '.cache': {}, + test: { + 'package.json': JSON.stringify({ + name: '@test/test', + }), + }, + 'cache.json': content, + }); + const lockfile = { + getDependencyTreeHash: () => 'test', + } as any as Lockfile; + const cache = await PackageDocsCache.loadAsync(testDir.path, lockfile); + expect(await cache.has('test')).toBe(false); }); }); diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts index 6b64b523e3..b8e22f65d6 100644 --- a/packages/repo-tools/src/commands/package-docs/Cache.ts +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -18,12 +18,11 @@ import globby from 'globby'; import { dirname, join as joinPath, relative } from 'path'; import crypto from 'crypto'; import { Lockfile } from '@backstage/cli-node'; -import { paths as cliPaths } from '../../lib/paths'; import { mkdirp } from 'fs-extra'; import { z } from 'zod'; +import { CACHE_DIR, CACHE_FILE } from './constants'; const version = '1'; -const CACHE_FILE = 'cache.json'; interface CacheEntry { hash: string; @@ -46,11 +45,13 @@ export class PackageDocsCache { private readonly lockfile: Lockfile, // A map of package directory to cache entry. private readonly cache: Map, - private readonly cacheDir: string, + private readonly baseDirectory: string, ) { this.keyCache = new Map(); } - static async loadAsync(cacheDir: string, lockfile: Lockfile) { + static async loadAsync(baseDirectory: string, lockfile: Lockfile) { + const cacheDir = joinPath(baseDirectory, CACHE_DIR); + await mkdirp(cacheDir); const cacheFiles = await globby(`**/${CACHE_FILE}`, { cwd: cacheDir, }); @@ -61,23 +62,29 @@ export class PackageDocsCache { try { const cacheJson = JSON.parse(cache); const parsed = cacheEntrySchema.parse(cacheJson); + if (parsed.version !== version) { + console.warn( + `Skipping cache file ${file} due to version mismatch: ${parsed.version} !== ${version}`, + ); + continue; + } map.set(pkg, parsed); } catch (e) { console.error(`Skipping unparseable cache file ${file}: ${e}`); } } - return new PackageDocsCache(lockfile, map, cacheDir); + return new PackageDocsCache(lockfile, map, baseDirectory); } async directoryToName(directory: string) { const packageJson = await readFile( - joinPath(directory, 'package.json'), + joinPath(this.baseDirectory, directory, 'package.json'), 'utf-8', ); return JSON.parse(packageJson).name; } - async toKey(pkg: string) { + private async toKey(pkg: string) { if (this.keyCache.has(pkg)) { return this.keyCache.get(pkg)!; } @@ -93,7 +100,7 @@ export class PackageDocsCache { hash.update('\0'); for (const path of result.sort()) { - const absPath = cliPaths.resolveTargetRoot(pkg, path); + const absPath = joinPath(this.baseDirectory, pkg, path); const pathInPackage = joinPath(absPath, path); hash.update(pathInPackage); hash.update('\0'); @@ -122,16 +129,16 @@ export class PackageDocsCache { } const cacheEntry = this.cache.get(pkg); const restoreTo = cacheEntry!.restoreTo; - const cacheDir = joinPath(this.cacheDir, pkg); + const cacheDir = joinPath(this.baseDirectory, CACHE_DIR, pkg); const contentsDir = joinPath(cacheDir, 'contents'); - const targetDir = cliPaths.resolveTargetRoot(restoreTo); + const targetDir = joinPath(this.baseDirectory, restoreTo); await mkdirp(targetDir); await cp(contentsDir, targetDir, { recursive: true }); } async write(pkg: string, contentDirectory: string) { - const cacheDir = joinPath(this.cacheDir, pkg); + const cacheDir = joinPath(this.baseDirectory, CACHE_DIR, pkg); const contentsDir = joinPath(cacheDir, 'contents'); await mkdirp(contentsDir); const hashString = await this.toKey(pkg); @@ -139,7 +146,7 @@ export class PackageDocsCache { const cacheEntry: CacheEntry = { hash: hashString, packageName: await this.directoryToName(pkg), - restoreTo: relative(cliPaths.resolveTargetRoot(), contentDirectory), + restoreTo: relative(this.baseDirectory, contentDirectory), version, }; await writeFile(joinPath(cacheDir, CACHE_FILE), JSON.stringify(cacheEntry)); diff --git a/packages/repo-tools/src/commands/package-docs/command.ts b/packages/repo-tools/src/commands/package-docs/command.ts index 50f52c6b87..d02bcad396 100644 --- a/packages/repo-tools/src/commands/package-docs/command.ts +++ b/packages/repo-tools/src/commands/package-docs/command.ts @@ -58,8 +58,6 @@ const HIGHLIGHT_LANGUAGES = [ 'json', ]; -const CACHE_DIR = '.cache/package-docs'; - function getExports(packageJson: any) { if (packageJson.exports) { return Object.values(packageJson.exports).filter( @@ -120,9 +118,8 @@ export default async function packageDocs(paths: string[] = [], opts: any) { exclude: opts.exclude, }); - await mkdirp(CACHE_DIR); const cache = await PackageDocsCache.loadAsync( - CACHE_DIR, + cliPaths.resolveTargetRoot(), await Lockfile.load(cliPaths.resolveTargetRoot('yarn.lock')), ); diff --git a/packages/repo-tools/src/commands/package-docs/constants.ts b/packages/repo-tools/src/commands/package-docs/constants.ts new file mode 100644 index 0000000000..a8372afad8 --- /dev/null +++ b/packages/repo-tools/src/commands/package-docs/constants.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const CACHE_FILE = 'cache.json'; +export const CACHE_DIR = '.cache/package-docs'; From 527e576145459f51760a8ec231404b409cf1cabb Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Thu, 8 May 2025 22:25:36 -0400 Subject: [PATCH 08/10] remove directory if it exists when writing Signed-off-by: aramissennyeydd --- packages/repo-tools/src/commands/package-docs/Cache.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/repo-tools/src/commands/package-docs/Cache.ts b/packages/repo-tools/src/commands/package-docs/Cache.ts index b8e22f65d6..487db81347 100644 --- a/packages/repo-tools/src/commands/package-docs/Cache.ts +++ b/packages/repo-tools/src/commands/package-docs/Cache.ts @@ -18,7 +18,7 @@ import globby from 'globby'; import { dirname, join as joinPath, relative } from 'path'; import crypto from 'crypto'; import { Lockfile } from '@backstage/cli-node'; -import { mkdirp } from 'fs-extra'; +import { exists, rm, mkdirp } from 'fs-extra'; import { z } from 'zod'; import { CACHE_DIR, CACHE_FILE } from './constants'; @@ -140,7 +140,11 @@ export class PackageDocsCache { async write(pkg: string, contentDirectory: string) { const cacheDir = joinPath(this.baseDirectory, CACHE_DIR, pkg); const contentsDir = joinPath(cacheDir, 'contents'); - await mkdirp(contentsDir); + if (await exists(contentsDir)) { + await rm(contentsDir, { recursive: true }); + } else { + await mkdirp(contentsDir); + } const hashString = await this.toKey(pkg); await cp(contentDirectory, contentsDir, { recursive: true }); const cacheEntry: CacheEntry = { From 6502efe8097e1d173dd5a8e49e5bed1f663eb41f Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Thu, 8 May 2025 22:27:54 -0400 Subject: [PATCH 09/10] ship .cache as default Signed-off-by: aramissennyeydd --- packages/create-app/templates/default-app/.gitignore.hbs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/create-app/templates/default-app/.gitignore.hbs b/packages/create-app/templates/default-app/.gitignore.hbs index 77ad56d128..78899bb081 100644 --- a/packages/create-app/templates/default-app/.gitignore.hbs +++ b/packages/create-app/templates/default-app/.gitignore.hbs @@ -52,3 +52,6 @@ site # E2E test reports e2e-test-report/ + +# Cache +.cache/ From 30474c43ae1369a081b81049a5d36309cf0b1521 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Thu, 8 May 2025 22:28:18 -0400 Subject: [PATCH 10/10] add changeset Signed-off-by: aramissennyeydd --- .changeset/salty-worlds-float.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/salty-worlds-float.md diff --git a/.changeset/salty-worlds-float.md b/.changeset/salty-worlds-float.md new file mode 100644 index 0000000000..a21d0b2e7c --- /dev/null +++ b/.changeset/salty-worlds-float.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': minor +--- + +Add .cache directory to shipped gitignore.