paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

add caching to the build to speed up incremental changes

Browse Source 
Signed-off-by: aramissennyeydd <aramis.sennyey@doordash.com>
This commit is contained in:
aramissennyeydd
2025-04-13 19:53:00 -04:00
parent c01c78b848
commit 52da6addc3
6 changed files with 276 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/deploy_microsite.yml
+36
View File
@@ -80,8 +80,18 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
continue-on-error: true
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -91,6 +101,14 @@ jobs:
if-no-files-found: error
retention-days: 1
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
- name: microsite yarn install
run: yarn install --immutable
working-directory: microsite
@@ -147,8 +165,18 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
continue-on-error: true
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -158,6 +186,14 @@ jobs:
if-no-files-found: error
retention-days: 1
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-${{ github.run_id }}
# Also build and upload storybook
- name: storybook yarn install
run: yarn install --immutable
.github/workflows/verify_microsite.yml
+36
View File
@@ -85,8 +85,18 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
continue-on-error: true
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -94,6 +104,14 @@ jobs:
name: stable-reference-beta
path: type-docs/
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-stable-${{ github.run_id }}
- name: microsite yarn install
run: yarn install --immutable
working-directory: microsite
@@ -149,8 +167,18 @@ jobs:
if-no-files-found: error
retention-days: 1
# Use the lower-level cache actions for the success cache, so that we can store the cache even on failed builds
- name: restore package-docs cache
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-
- name: build API reference (beta)
run: yarn backstage-repo-tools package-docs
continue-on-error: true
- name: upload API reference (beta)
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -158,6 +186,14 @@ jobs:
name: next-reference-beta
path: type-docs/
# Always save success cache even if there were failures, that way it can be used in re-triggered builds
- name: save package-docs cache
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4
if: always()
with:
path: .cache/package-docs
key: ${{ runner.os }}-v${{ matrix.node-version }}-package-docs-next-${{ github.run_id }}
# Also build and upload storybook
- name: storybook yarn install
run: yarn install --immutable
packages/repo-tools/package.json
+1
View File
@@ -69,6 +69,7 @@
"commander": "^12.0.0",
"fs-extra": "^11.2.0",
"glob": "^8.0.3",
"globby": "^11.0.0",
"is-glob": "^4.0.3",
"js-yaml": "^4.1.0",
"just-diff": "^6.0.2",
packages/repo-tools/src/commands/package-docs/Cache.ts
+140
View File
@@ -0,0 +1,140 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { readFile, writeFile, cp } from 'fs/promises';
import globby from 'globby';
import workerPath, { dirname } from 'path';
import crypto from 'crypto';
import { Lockfile } from '@backstage/cli-node';
import { paths as cliPaths } from '../../lib/paths';
import { mkdirp } from 'fs-extra';
const version = '1';
const CACHE_FILE = 'cache.json';
interface CacheEntry {
hash: string;
packageName: string;
restoreTo: string;
version: string;
}
export class PackageDocsCache {
private keyCache: Map<string, string>;
constructor(
private readonly lockfile: Lockfile,
private readonly cache: Map<string, CacheEntry>,
private readonly cacheDir: string,
) {
this.keyCache = new Map();
}
static async loadAsync(cacheDir: string, lockfile: Lockfile) {
const cacheFiles = await globby(`**/${CACHE_FILE}`, {
cwd: cacheDir,
});
const map = new Map<string, CacheEntry>();
for (const file of cacheFiles) {
console.log(file);
const pkg = dirname(file);
const cache = await readFile(workerPath.join(cacheDir, file), 'utf-8');
const cacheJson = JSON.parse(cache);
map.set(pkg, cacheJson);
}
return new PackageDocsCache(lockfile, map, cacheDir);
}
async directoryToName(directory: string) {
const packageJson = await readFile(
workerPath.join(directory, 'package.json'),
'utf-8',
);
return JSON.parse(packageJson).name;
}
async toKey(pkg: string) {
if (this.keyCache.has(pkg)) {
return this.keyCache.get(pkg)!;
}
const name = await this.directoryToName(pkg);
const result = await globby('src/**', {
gitignore: true,
onlyFiles: true,
cwd: pkg,
});
const hash = crypto.createHash('sha1');
hash.update(version);
hash.update('\0');
for (const path of result.sort()) {
const absPath = cliPaths.resolveTargetRoot(pkg, path);
const pathInPackage = workerPath.join(absPath, path);
hash.update(pathInPackage);
hash.update('\0');
hash.update(await readFile(absPath));
hash.update('\0');
}
hash.update(this.lockfile.getDependencyTreeHash(name));
hash.update('\0');
const hashString = hash.digest('hex');
this.keyCache.set(pkg, hashString);
return hashString;
}
async has(pkg: string) {
const cache = this.cache.get(pkg);
if (!cache) {
return false;
}
const hashString = await this.toKey(pkg);
return cache.hash === hashString;
}
async restore(pkg: string) {
if (!this.has(pkg)) {
throw new Error(`Cache entry for ${pkg} not found`);
}
const cacheEntry = this.cache.get(pkg);
const restoreTo = cacheEntry!.restoreTo;
const cacheDir = workerPath.join(this.cacheDir, pkg);
const contentsDir = workerPath.join(cacheDir, 'contents');
const targetDir = cliPaths.resolveTargetRoot(restoreTo);
await mkdirp(targetDir);
console.log(contentsDir, targetDir);
await cp(contentsDir, targetDir, { recursive: true });
}
async write(pkg: string, contentDirectory: string) {
const cacheDir = workerPath.join(this.cacheDir, pkg);
const contentsDir = workerPath.join(cacheDir, 'contents');
await mkdirp(cacheDir);
const hashString = await this.toKey(pkg);
await cp(contentDirectory, contentsDir, { recursive: true });
const cacheEntry: CacheEntry = {
hash: hashString,
packageName: await this.directoryToName(pkg),
restoreTo: workerPath.relative(
cliPaths.resolveTargetRoot(),
contentDirectory,
),
version,
};
await writeFile(
workerPath.join(cacheDir, CACHE_FILE),
JSON.stringify(cacheEntry),
);
}
}
packages/repo-tools/src/commands/package-docs/command.ts
+62 -35
View File
@@ -17,8 +17,11 @@ import { exec } from 'child_process';
import { promisify } from 'util';
import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
import { createTemporaryTsConfig } from './utils';
import { mkdir, readFile, writeFile } from 'fs/promises';
import { readFile, writeFile } from 'fs/promises';
import pLimit from 'p-limit';
import { mkdirp } from 'fs-extra';
import { PackageDocsCache } from './Cache';
import { Lockfile } from '@backstage/cli-node';
const limit = pLimit(8);
@@ -38,6 +41,7 @@ const EXCLUDE = [
'packages/techdocs-cli-embedded-app',
'packages/yarn-plugin',
'packages/backend',
'packages/backend-legacy',
];
const HIGHLIGHT_LANGUAGES = [
@@ -54,6 +58,8 @@ const HIGHLIGHT_LANGUAGES = [
'json',
];
const CACHE_DIR = '.cache/package-docs';
function getExports(packageJson: any) {
if (packageJson.exports) {
return Object.values(packageJson.exports).filter(
@@ -75,41 +81,35 @@ async function generateDocJson(pkg: string) {
!exports.length ||
!exports.some(e => e.startsWith('src') || e.startsWith('./src'))
) {
return;
return false;
}
try {
await mkdir(cliPaths.resolveTargetRoot(`dist-types`, pkg), {
recursive: true,
});
await mkdirp(cliPaths.resolveTargetRoot(`dist-types`, pkg));
const { stdout, stderr } = await execAsync(
[
cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
'--json',
cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'),
'--tsconfig',
temporaryTsConfigPath,
'--basePath',
cliPaths.targetRoot,
'--skipErrorChecking',
...(getExports(packageJson).flatMap(e => [
'--entryPoints',
e,
]) as string[]),
].join(' '),
{
cwd: pkg,
env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
},
);
console.log(`### Processed ${pkg}`);
console.log(stdout);
console.error(stderr);
} catch (e) {
console.error('Failed to generate docs for', pkg);
console.error(e);
}
const { stdout, stderr } = await execAsync(
[
cliPaths.resolveTargetRoot('node_modules/.bin/typedoc'),
'--json',
cliPaths.resolveTargetRoot(`dist-types`, pkg, 'docs.json'),
'--tsconfig',
temporaryTsConfigPath,
'--basePath',
cliPaths.targetRoot,
'--skipErrorChecking',
...(getExports(packageJson).flatMap(e => [
'--entryPoints',
e,
]) as string[]),
].join(' '),
{
cwd: pkg,
env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=12288' },
},
);
console.log(`### Processed ${pkg}`);
console.log(stdout);
console.error(stderr);
return true;
}
export default async function packageDocs(paths: string[] = [], opts: any) {
@@ -120,6 +120,12 @@ export default async function packageDocs(paths: string[] = [], opts: any) {
exclude: opts.exclude,
});
await mkdirp(CACHE_DIR);
const cache = await PackageDocsCache.loadAsync(
CACHE_DIR,
await Lockfile.load(cliPaths.resolveTargetRoot('yarn.lock')),
);
console.log(`### Generating docs.`);
await Promise.all(
selectedPackageDirs.map(pkg =>
@@ -127,8 +133,29 @@ export default async function packageDocs(paths: string[] = [], opts: any) {
if (EXCLUDE.includes(pkg)) {
return;
}
console.log(`### Processing ${pkg}`);
await generateDocJson(pkg);
if (await cache.has(pkg)) {
console.log(`### Skipping ${pkg} due to cache hit`);
try {
await cache.restore(pkg);
return;
} catch (e) {
console.error('Failed to restore cache for', pkg);
console.error(e);
}
}
try {
console.log(`### Processing ${pkg}`);
const success = await generateDocJson(pkg);
if (success) {
await cache.write(
pkg,
cliPaths.resolveTargetRoot(`dist-types`, pkg),
);
}
} catch (e) {
console.error('Failed to generate docs for', pkg);
console.error(e);
}
}),
),
);
yarn.lock
+1
View File
@@ -8414,6 +8414,7 @@ __metadata:
commander: "npm:^12.0.0"
fs-extra: "npm:^11.2.0"
glob: "npm:^8.0.3"
globby: "npm:^11.0.0"
is-glob: "npm:^4.0.3"
js-yaml: "npm:^4.1.0"
just-diff: "npm:^6.0.2"