From f2a589777ec8768a11af013db3f9c9b63651a698 Mon Sep 17 00:00:00 2001 From: Peter Macdonald Date: Wed, 18 Sep 2024 11:20:50 +0200 Subject: [PATCH 01/20] Sets the edit this page url to be static as it was before we generated versioned docs Signed-off-by: Peter Macdonald --- microsite/docusaurus.config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/microsite/docusaurus.config.ts b/microsite/docusaurus.config.ts index af96ef4c42..43bde19e35 100644 --- a/microsite/docusaurus.config.ts +++ b/microsite/docusaurus.config.ts @@ -84,7 +84,10 @@ const config: Config = { /** @type {import('@docusaurus/preset-classic').Options} */ { docs: { - editUrl: 'https://github.com/backstage/backstage/edit/master/docs/', + editUrl: ({ docPath }) => { + // Always point to the non-versioned directory when editing a doc page + return `https://github.com/backstage/backstage/edit/master/docs/${docPath}`; + }, path: '../docs', sidebarPath: 'sidebars.js', ...(useVersionedDocs From b084f5aacc6b4f0eb222fd703c7db7346674fd06 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 16 Oct 2024 13:18:59 +0200 Subject: [PATCH 02/20] cli: bump webpack dep range Signed-off-by: Patrik Oldsberg --- .changeset/wild-experts-unite.md | 5 +++++ packages/cli/package.json | 2 +- yarn.lock | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changeset/wild-experts-unite.md diff --git a/.changeset/wild-experts-unite.md b/.changeset/wild-experts-unite.md new file mode 100644 index 0000000000..872a3aa8de --- /dev/null +++ b/.changeset/wild-experts-unite.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Bump the Webpack dependency range to `^5.94.0`, as our current configuration is not compatible with some older versions. diff --git a/packages/cli/package.json b/packages/cli/package.json index fb5597b8ec..b61d3c7149 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -149,7 +149,7 @@ "terser-webpack-plugin": "^5.1.3", "ts-morph": "^23.0.0", "util": "^0.12.3", - "webpack": "^5.70.0", + "webpack": "^5.94.0", "webpack-dev-server": "^5.0.0", "yaml": "^2.0.0", "yargs": "^16.2.0", diff --git a/yarn.lock b/yarn.lock index 8567fd5fa5..e4876c662c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4029,7 +4029,7 @@ __metadata: vite: ^5.0.0 vite-plugin-html: ^3.2.2 vite-plugin-node-polyfills: ^0.22.0 - webpack: ^5.70.0 + webpack: ^5.94.0 webpack-dev-server: ^5.0.0 yaml: ^2.0.0 yargs: ^16.2.0 @@ -44704,7 +44704,7 @@ __metadata: languageName: node linkType: hard -"webpack@npm:^5, webpack@npm:^5.70.0": +"webpack@npm:^5, webpack@npm:^5.94.0": version: 5.95.0 resolution: "webpack@npm:5.95.0" dependencies: From 683dae27182d5ae7a8d1f4e9f75c64711e107d1d Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Tue, 19 Mar 2024 10:21:09 +0100 Subject: [PATCH 03/20] starting watch command Signed-off-by: aramissennyeydd --- .../src/generated/.openapi-generator-ignore | 24 ++++++++++++ .../src/generated/apis/DefaultApi.client.ts | 9 +++-- packages/repo-tools/package.json | 1 + packages/repo-tools/src/commands/index.ts | 2 + .../package/schema/openapi/generate/client.ts | 14 +++++-- .../package/schema/openapi/generate/index.ts | 39 ++++++++++++++++--- .../package/schema/openapi/generate/server.ts | 17 +++++--- yarn.lock | 1 + 8 files changed, 91 insertions(+), 16 deletions(-) create mode 100644 packages/catalog-client/src/generated/.openapi-generator-ignore diff --git a/packages/catalog-client/src/generated/.openapi-generator-ignore b/packages/catalog-client/src/generated/.openapi-generator-ignore new file mode 100644 index 0000000000..bd8a6eec07 --- /dev/null +++ b/packages/catalog-client/src/generated/.openapi-generator-ignore @@ -0,0 +1,24 @@ +*.md +*.mustache +apis/baseapi.ts +apis/exception.ts +auth/* +http/* +middleware.ts +servers.ts +util.ts +configuration.ts +rxjsStub.ts +.gitignore +apis/*.ts +!apis/*.client.ts +models/*.ts +!models/*.model.ts +!index.ts +!**/index.ts +types/ObjectParamAPI.ts +types/ObservableAPI.ts +types/PromiseAPI.ts +git_push.sh +package.json +tsconfig.json \ No newline at end of file diff --git a/packages/catalog-client/src/generated/apis/DefaultApi.client.ts b/packages/catalog-client/src/generated/apis/DefaultApi.client.ts index 1f1c54d6f8..598d9c706f 100644 --- a/packages/catalog-client/src/generated/apis/DefaultApi.client.ts +++ b/packages/catalog-client/src/generated/apis/DefaultApi.client.ts @@ -276,13 +276,14 @@ export class DefaultApiClient { /** * Get a batch set of entities given an array of entityRefs. + * @param filter Filter for just the entities defined by this filter. * @param getEntitiesByRefsRequest */ public async getEntitiesByRefs( // @ts-ignore request: { body: GetEntitiesByRefsRequest; - query?: { + query: { filter?: Array; }; }, @@ -290,9 +291,11 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/entities/by-refs/{?filter*}`; + const uriTemplate = `/entities/by-refs{?filter*}`; - const uri = parser.parse(uriTemplate).expand({ ...request.query }); + const uri = parser.parse(uriTemplate).expand({ + ...request.query, + }); return await this.fetchApi.fetch(`${baseUrl}${uri}`, { headers: { diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index f4b5133b30..a8fef81b63 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -62,6 +62,7 @@ "@stoplight/types": "^14.0.0", "@useoptic/openapi-utilities": "^0.55.0", "chalk": "^4.0.0", + "chokidar": "^3.6.0", "codeowners-utils": "^1.0.2", "command-exists": "^1.2.9", "commander": "^12.0.0", diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index 5a69a74ff3..abe55356ef 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -58,6 +58,8 @@ function registerPackageCommand(program: Command) { .description( 'Additional properties that can be passed to @openapitools/openapi-generator-cli', ) + .option('--watch') + .description('Watch the OpenAPI spec for changes and regenerate on save.') .action( lazy(() => import('./package/schema/openapi/generate').then(m => m.command), diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts index b843cd27ec..61be02979e 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts @@ -30,6 +30,7 @@ import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers' async function generate( outputDirectory: string, clientAdditionalProperties?: string, + abortSignal?: AbortController, ) { const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); const resolvedOutputDirectory = cliPaths.resolveTargetRoot( @@ -69,6 +70,7 @@ async function generate( additionalProperties, ], { + signal: abortSignal?.signal, maxBuffer: Number.MAX_VALUE, cwd: resolvePackagePath('@backstage/repo-tools'), env: { @@ -83,7 +85,9 @@ async function generate( const prettier = cliPaths.resolveTargetRoot('node_modules/.bin/prettier'); if (prettier) { - await exec(`${prettier} --write ${resolvedOutputDirectory}`); + await exec(`${prettier} --write ${resolvedOutputDirectory}`, [], { + signal: abortSignal?.signal, + }); } fs.removeSync(resolve(resolvedOutputDirectory, '.openapi-generator-ignore')); @@ -97,14 +101,18 @@ async function generate( export async function command( outputPackage: string, clientAdditionalProperties?: string, + abortSignal?: AbortController, ): Promise { try { - await generate(outputPackage, clientAdditionalProperties); + await generate(outputPackage, clientAdditionalProperties, abortSignal); console.log( chalk.green(`Generated client in ${outputPackage}/${OUTPUT_PATH}`), ); } catch (err) { - console.log(); + if (err.name === 'AbortError') { + console.debug('Server generation aborted.'); + return; + } console.log(chalk.red(`Client generation failed:`)); console.log(err); diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts index 41884e2e90..9227b67ba8 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts @@ -17,6 +17,8 @@ import chalk from 'chalk'; import { OptionValues } from 'commander'; import { command as generateClient } from './client'; import { command as generateServer } from './server'; +import chokidar from 'chokidar'; +import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'; export async function command(opts: OptionValues) { if (!opts.clientPackage && !opts.server) { @@ -25,10 +27,37 @@ export async function command(opts: OptionValues) { ); process.exit(1); } - if (opts.clientPackage) { - await generateClient(opts.clientPackage, opts.clientAdditionalProperties); - } - if (opts.server) { - await generateServer(); + + const sharedCommand = async (abortSignal?: AbortController) => { + if (opts.clientPackage) { + await generateClient( + opts.clientPackage, + opts.clientAdditionalProperties, + abortSignal, + ); + } + if (opts.server) { + await generateServer(abortSignal); + } + }; + + if (opts.watch) { + try { + const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); + let abortController = new AbortController(); + chokidar.watch(resolvedOpenapiPath).on('change', async () => { + console.log('detected changes'); + abortController.abort(); + await sharedCommand(abortController); + abortController = new AbortController(); + }); + await sharedCommand(); + await new Promise(() => {}); + } catch (err) { + console.error(chalk.red('Error: ', err)); + process.exit(1); + } + } else { + await sharedCommand(); } } diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts index e6c56c3122..317c17f96d 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts @@ -25,7 +25,7 @@ import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers' const exec = promisify(execCb); -async function generate() { +async function generate(abortSignal?: AbortController) { const openapiPath = await getPathToCurrentOpenApiSpec(); const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8')); @@ -50,19 +50,26 @@ export const createOpenApiRouter = async ( `, ); - await exec(`yarn backstage-cli package lint --fix ${tsPath}`); + await exec(`yarn backstage-cli package lint --fix ${tsPath}`, { + signal: abortSignal?.signal, + }); if (await cliPaths.resolveTargetRoot('node_modules/.bin/prettier')) { await exec(`yarn prettier --write ${tsPath}`, { cwd: cliPaths.targetRoot, + signal: abortSignal?.signal, }); } } -export async function command(): Promise { +export async function command(abortSignal?: AbortController): Promise { try { - await generate(); - console.log(chalk.green('Generated all files.')); + await generate(abortSignal); + console.log(chalk.green('Generated server files.')); } catch (err) { + if (err.name === 'AbortError') { + console.debug('Server generation aborted.'); + return; + } console.log(chalk.red(`OpenAPI server stub generation failed.`)); console.log(err.message); process.exit(1); diff --git a/yarn.lock b/yarn.lock index d8070957c2..eba469581e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8309,6 +8309,7 @@ __metadata: "@types/prettier": ^2.0.0 "@useoptic/openapi-utilities": ^0.55.0 chalk: ^4.0.0 + chokidar: ^3.6.0 codeowners-utils: ^1.0.2 command-exists: ^1.2.9 commander: ^12.0.0 From 3fb2fd76559213742bdbebdde5faa7880d4df0f1 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 7 Jul 2024 17:36:45 -0400 Subject: [PATCH 04/20] implement watch command Signed-off-by: aramissennyeydd --- .../src/generated/.openapi-generator-ignore | 24 -------------- packages/repo-tools/package.json | 2 +- .../package/schema/openapi/generate/index.ts | 32 +++++++++++++------ yarn.lock | 2 +- 4 files changed, 25 insertions(+), 35 deletions(-) delete mode 100644 packages/catalog-client/src/generated/.openapi-generator-ignore diff --git a/packages/catalog-client/src/generated/.openapi-generator-ignore b/packages/catalog-client/src/generated/.openapi-generator-ignore deleted file mode 100644 index bd8a6eec07..0000000000 --- a/packages/catalog-client/src/generated/.openapi-generator-ignore +++ /dev/null @@ -1,24 +0,0 @@ -*.md -*.mustache -apis/baseapi.ts -apis/exception.ts -auth/* -http/* -middleware.ts -servers.ts -util.ts -configuration.ts -rxjsStub.ts -.gitignore -apis/*.ts -!apis/*.client.ts -models/*.ts -!models/*.model.ts -!index.ts -!**/index.ts -types/ObjectParamAPI.ts -types/ObservableAPI.ts -types/PromiseAPI.ts -git_push.sh -package.json -tsconfig.json \ No newline at end of file diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index a8fef81b63..620456443d 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -62,7 +62,7 @@ "@stoplight/types": "^14.0.0", "@useoptic/openapi-utilities": "^0.55.0", "chalk": "^4.0.0", - "chokidar": "^3.6.0", + "chokidar": "^3.5.3", "codeowners-utils": "^1.0.2", "command-exists": "^1.2.9", "commander": "^12.0.0", diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts index 9227b67ba8..b10ae941d4 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts @@ -29,29 +29,43 @@ export async function command(opts: OptionValues) { } const sharedCommand = async (abortSignal?: AbortController) => { + const promises = []; if (opts.clientPackage) { - await generateClient( - opts.clientPackage, - opts.clientAdditionalProperties, - abortSignal, + promises.push( + generateClient( + opts.clientPackage, + opts.clientAdditionalProperties, + abortSignal, + ), ); } if (opts.server) { - await generateServer(abortSignal); + promises.push(generateServer(abortSignal)); } + await Promise.all(promises); }; if (opts.watch) { try { const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); let abortController = new AbortController(); - chokidar.watch(resolvedOpenapiPath).on('change', async () => { - console.log('detected changes'); + const watcher = chokidar.watch(resolvedOpenapiPath); + watcher.on('change', () => { + console.log('Detected changes! Regenerating...'); abortController.abort(); - await sharedCommand(abortController); abortController = new AbortController(); + sharedCommand(abortController); + }); + watcher.on('error', error => { + console.error('Error happened', error); + }); + + watcher.on('ready', async () => { + console.log( + 'Watching for changes in OpenAPI spec. Press Ctrl+C to stop.', + ); + await sharedCommand(); }); - await sharedCommand(); await new Promise(() => {}); } catch (err) { console.error(chalk.red('Error: ', err)); diff --git a/yarn.lock b/yarn.lock index eba469581e..af7d13981a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8309,7 +8309,7 @@ __metadata: "@types/prettier": ^2.0.0 "@useoptic/openapi-utilities": ^0.55.0 chalk: ^4.0.0 - chokidar: ^3.6.0 + chokidar: ^3.5.3 codeowners-utils: ^1.0.2 command-exists: ^1.2.9 commander: ^12.0.0 From 568cfdd9a9879d5a39c7fb3f68074b7e145cff29 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 7 Jul 2024 17:55:09 -0400 Subject: [PATCH 05/20] improve error handling so that the main file process.exits Signed-off-by: aramissennyeydd --- .../package/schema/openapi/generate/client.ts | 4 ++-- .../package/schema/openapi/generate/index.ts | 20 +++++++++++++++---- .../package/schema/openapi/generate/server.ts | 1 - 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts index 61be02979e..66f750bfc3 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts @@ -81,6 +81,8 @@ async function generate( await exec( `yarn backstage-cli package lint --fix ${resolvedOutputDirectory}`, + [], + { signal: abortSignal?.signal }, ); const prettier = cliPaths.resolveTargetRoot('node_modules/.bin/prettier'); @@ -115,7 +117,5 @@ export async function command( } console.log(chalk.red(`Client generation failed:`)); console.log(err); - - process.exit(1); } } diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts index b10ae941d4..a7e9d9a402 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts @@ -19,6 +19,7 @@ import { command as generateClient } from './client'; import { command as generateServer } from './server'; import chokidar from 'chokidar'; import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'; +import { debounce } from 'lodash'; export async function command(opts: OptionValues) { if (!opts.clientPackage && !opts.server) { @@ -50,11 +51,19 @@ export async function command(opts: OptionValues) { const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); let abortController = new AbortController(); const watcher = chokidar.watch(resolvedOpenapiPath); - watcher.on('change', () => { + + // The generate command currently takes ~8 seconds to run, so let's debounce calling it so we don't have to cancel it so much. + const debouncedCommand = debounce(() => { console.log('Detected changes! Regenerating...'); abortController.abort(); abortController = new AbortController(); - sharedCommand(abortController); + sharedCommand(abortController).catch(err => { + console.error(chalk.red('Error: ', err)); + }); + }, 500); + + watcher.on('change', () => { + debouncedCommand(); }); watcher.on('error', error => { console.error('Error happened', error); @@ -64,7 +73,6 @@ export async function command(opts: OptionValues) { console.log( 'Watching for changes in OpenAPI spec. Press Ctrl+C to stop.', ); - await sharedCommand(); }); await new Promise(() => {}); } catch (err) { @@ -72,6 +80,10 @@ export async function command(opts: OptionValues) { process.exit(1); } } else { - await sharedCommand(); + try { + await sharedCommand(); + } catch (err) { + process.exit(1); + } } } diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts index 317c17f96d..662efe4347 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts @@ -72,6 +72,5 @@ export async function command(abortSignal?: AbortController): Promise { } console.log(chalk.red(`OpenAPI server stub generation failed.`)); console.log(err.message); - process.exit(1); } } From 47fdbb4d96a22021f2f6463ab6fc4bb48bbce098 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Sun, 7 Jul 2024 17:58:34 -0400 Subject: [PATCH 06/20] add changeset and revert generated file change Signed-off-by: aramissennyeydd --- .changeset/slow-mayflies-sleep.md | 5 +++++ .../src/generated/apis/DefaultApi.client.ts | 9 +++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 .changeset/slow-mayflies-sleep.md diff --git a/.changeset/slow-mayflies-sleep.md b/.changeset/slow-mayflies-sleep.md new file mode 100644 index 0000000000..ebfb9d12d4 --- /dev/null +++ b/.changeset/slow-mayflies-sleep.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': minor +--- + +Adds a `--watch` mode to the `schema openapi generate` command for a better local schema writing experience. diff --git a/packages/catalog-client/src/generated/apis/DefaultApi.client.ts b/packages/catalog-client/src/generated/apis/DefaultApi.client.ts index 598d9c706f..1f1c54d6f8 100644 --- a/packages/catalog-client/src/generated/apis/DefaultApi.client.ts +++ b/packages/catalog-client/src/generated/apis/DefaultApi.client.ts @@ -276,14 +276,13 @@ export class DefaultApiClient { /** * Get a batch set of entities given an array of entityRefs. - * @param filter Filter for just the entities defined by this filter. * @param getEntitiesByRefsRequest */ public async getEntitiesByRefs( // @ts-ignore request: { body: GetEntitiesByRefsRequest; - query: { + query?: { filter?: Array; }; }, @@ -291,11 +290,9 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/entities/by-refs{?filter*}`; + const uriTemplate = `/entities/by-refs/{?filter*}`; - const uri = parser.parse(uriTemplate).expand({ - ...request.query, - }); + const uri = parser.parse(uriTemplate).expand({ ...request.query }); return await this.fetchApi.fetch(`${baseUrl}${uri}`, { headers: { From abc3e3475cd8b0182587bb2a76f97e17572e90bb Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Mon, 7 Oct 2024 23:17:58 -0400 Subject: [PATCH 07/20] add a unit test Signed-off-by: aramissennyeydd --- .../schema/openapi/generate/index.test.ts | 103 ++++++++++++++++++ .../package/schema/openapi/generate/index.ts | 3 +- packages/repo-tools/src/lib/runner.ts | 4 + 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts new file mode 100644 index 0000000000..21a11c6165 --- /dev/null +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createMockDirectory } from '@backstage/backend-test-utils'; +import path from 'path'; + +jest.mock( + 'lodash', + () => + ({ + ...jest.requireActual('lodash'), + debounce: (fn: any) => fn, + } as any as typeof import('lodash')), +); + +describe('generateOpenApiSchema', () => { + const inputDir = createMockDirectory(); + const outputDir = createMockDirectory(); + + beforeEach(() => { + inputDir.clear(); + outputDir.clear(); + + inputDir.addContent({ + 'openapi.yaml': ` + openapi: 3.0.0 + info: + title: Test API + version: 1.0.0 + paths: + /test: + get: + responses: + '200': + description: OK + `, + }); + jest.mock('../../../../../lib/openapi/helpers', () => ({ + getPathToCurrentOpenApiSpec: jest.fn(() => + Promise.resolve(path.join(inputDir.path, 'openapi.yaml')), + ), + })); + }); + it('should handle watch mode', async () => { + const generateClientMock = jest.fn(); + const generateServerMock = jest.fn(); + jest.doMock('./client', () => ({ command: generateClientMock })); + jest.doMock('./server', () => ({ command: generateServerMock })); + + const mockWatch = jest.fn(); + jest.mock('chokidar', () => ({ + watch: mockWatch, + })); + let resolve: (val?: any) => void; + const block = () => + new Promise(res => { + resolve = res; + }); + jest.mock('../../../../../lib/runner', () => ({ + block, + })); + + const mockOn: Record = {}; + mockWatch.mockReturnValue({ + on: jest.fn((event, cb) => { + console.log(event); + mockOn[event] = cb; + }), + } as any); + + const { command } = await import('./index'); + + const actions = async () => { + while (!mockOn.ready) { + // Wait for the watcher to be registered + await new Promise(r => setTimeout(r, 100)); + } + + expect(generateClientMock).toHaveBeenCalledTimes(0); + mockOn.change(); + expect(generateClientMock).toHaveBeenCalledTimes(1); + resolve(); + }; + + await Promise.all([ + actions(), + command({ watch: true, clientPackage: 'test123' }), + ]); + }); +}); diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts index a7e9d9a402..e8a7eb7249 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts @@ -20,6 +20,7 @@ import { command as generateServer } from './server'; import chokidar from 'chokidar'; import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'; import { debounce } from 'lodash'; +import { block } from '../../../../../lib/runner'; export async function command(opts: OptionValues) { if (!opts.clientPackage && !opts.server) { @@ -74,7 +75,7 @@ export async function command(opts: OptionValues) { 'Watching for changes in OpenAPI spec. Press Ctrl+C to stop.', ); }); - await new Promise(() => {}); + await block(); } catch (err) { console.error(chalk.red('Error: ', err)); process.exit(1); diff --git a/packages/repo-tools/src/lib/runner.ts b/packages/repo-tools/src/lib/runner.ts index ea7b511938..8389caa63a 100644 --- a/packages/repo-tools/src/lib/runner.ts +++ b/packages/repo-tools/src/lib/runner.ts @@ -67,3 +67,7 @@ export async function runner( return resultsList; } + +export async function block() { + return new Promise(() => {}); +} From 879e4cb45abf34f42de3a272130a5a03945cf081 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Tue, 15 Oct 2024 21:16:39 -0400 Subject: [PATCH 08/20] fix error UX Signed-off-by: aramissennyeydd --- .../package/schema/openapi/generate/client.ts | 16 +++++++++++++--- .../package/schema/openapi/generate/index.ts | 17 ++++++++++++++--- .../package/schema/openapi/generate/server.ts | 19 ++++++++++++++++--- .../commands/repo/schema/openapi/verify.ts | 12 ++++++------ .../repo-tools/src/lib/openapi/helpers.ts | 10 ++++++++++ 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts index 66f750bfc3..742f303f51 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts @@ -103,7 +103,10 @@ async function generate( export async function command( outputPackage: string, clientAdditionalProperties?: string, - abortSignal?: AbortController, + { + abortSignal, + isWatch = false, + }: { abortSignal?: AbortController; isWatch?: boolean } = {}, ): Promise { try { await generate(outputPackage, clientAdditionalProperties, abortSignal); @@ -115,7 +118,14 @@ export async function command( console.debug('Server generation aborted.'); return; } - console.log(chalk.red(`Client generation failed:`)); - console.log(err); + if (isWatch) { + console.log(chalk.red(`Client generation failed:`)); + console.group(); + console.log(chalk.red(err.message)); + console.groupEnd(); + } else { + console.log(chalk.red(`Client generation failed.`)); + console.log(chalk.red(err.message)); + } } } diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts index e8a7eb7249..1c9fd586fb 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts @@ -18,7 +18,10 @@ import { OptionValues } from 'commander'; import { command as generateClient } from './client'; import { command as generateServer } from './server'; import chokidar from 'chokidar'; -import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'; +import { + getPathToCurrentOpenApiSpec, + loadAndValidateOpenApiYaml, +} from '../../../../../lib/openapi/helpers'; import { debounce } from 'lodash'; import { block } from '../../../../../lib/runner'; @@ -31,18 +34,24 @@ export async function command(opts: OptionValues) { } const sharedCommand = async (abortSignal?: AbortController) => { + const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); + await loadAndValidateOpenApiYaml(resolvedOpenapiPath); const promises = []; + const options = { + isWatch: opts.watch, + abortSignal, + }; if (opts.clientPackage) { promises.push( generateClient( opts.clientPackage, opts.clientAdditionalProperties, - abortSignal, + options, ), ); } if (opts.server) { - promises.push(generateServer(abortSignal)); + promises.push(generateServer(options)); } await Promise.all(promises); }; @@ -75,6 +84,8 @@ export async function command(opts: OptionValues) { 'Watching for changes in OpenAPI spec. Press Ctrl+C to stop.', ); }); + + debouncedCommand(); await block(); } catch (err) { console.error(chalk.red('Error: ', err)); diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts index 662efe4347..1ef6fd10d2 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts @@ -61,7 +61,13 @@ export const createOpenApiRouter = async ( } } -export async function command(abortSignal?: AbortController): Promise { +export async function command({ + abortSignal, + isWatch = false, +}: { + abortSignal?: AbortController; + isWatch?: boolean; +}): Promise { try { await generate(abortSignal); console.log(chalk.green('Generated server files.')); @@ -70,7 +76,14 @@ export async function command(abortSignal?: AbortController): Promise { console.debug('Server generation aborted.'); return; } - console.log(chalk.red(`OpenAPI server stub generation failed.`)); - console.log(err.message); + if (isWatch) { + console.log(chalk.red(`Server generation failed:`)); + console.group(); + console.log(chalk.red(err.message)); + console.groupEnd(); + } else { + console.log(chalk.red(err.message)); + console.log(chalk.red(`OpenAPI server stub generation failed.`)); + } } } diff --git a/packages/repo-tools/src/commands/repo/schema/openapi/verify.ts b/packages/repo-tools/src/commands/repo/schema/openapi/verify.ts index 23e07d1230..35dd3e1702 100644 --- a/packages/repo-tools/src/commands/repo/schema/openapi/verify.ts +++ b/packages/repo-tools/src/commands/repo/schema/openapi/verify.ts @@ -15,12 +15,10 @@ */ import fs from 'fs-extra'; -import YAML from 'js-yaml'; -import { isEqual, cloneDeep } from 'lodash'; +import { isEqual } from 'lodash'; import { join } from 'path'; import chalk from 'chalk'; import { relative as relativePath, resolve as resolvePath } from 'path'; -import Parser from '@apidevtools/swagger-parser'; import { runner } from '../../../../lib/runner'; import { paths as cliPaths } from '../../../../lib/paths'; import { @@ -28,7 +26,10 @@ import { TS_SCHEMA_PATH, YAML_SCHEMA_PATH, } from '../../../../lib/openapi/constants'; -import { getPathToOpenApiSpec } from '../../../../lib/openapi/helpers'; +import { + getPathToOpenApiSpec, + loadAndValidateOpenApiYaml, +} from '../../../../lib/openapi/helpers'; async function verify(directoryPath: string) { let openapiPath = ''; @@ -38,8 +39,7 @@ async function verify(directoryPath: string) { // Unable to find spec at path. return; } - const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8')); - await Parser.validate(cloneDeep(yaml) as any); + const yaml = await loadAndValidateOpenApiYaml(openapiPath); const schemaPath = join(directoryPath, TS_SCHEMA_PATH); if (!(await fs.pathExists(schemaPath))) { diff --git a/packages/repo-tools/src/lib/openapi/helpers.ts b/packages/repo-tools/src/lib/openapi/helpers.ts index f7e684a407..2359a1bbb0 100644 --- a/packages/repo-tools/src/lib/openapi/helpers.ts +++ b/packages/repo-tools/src/lib/openapi/helpers.ts @@ -18,6 +18,10 @@ import { pathExists } from 'fs-extra'; import { paths } from '../paths'; import { YAML_SCHEMA_PATH } from './constants'; import { resolve } from 'path'; +import YAML from 'js-yaml'; +import { cloneDeep } from 'lodash'; +import Parser from '@apidevtools/swagger-parser'; +import fs from 'fs-extra'; export const getPathToFile = async (directory: string, filename: string) => { return resolve(directory, filename); @@ -41,3 +45,9 @@ export const getPathToOpenApiSpec = async (directory: string) => { export const getPathToCurrentOpenApiSpec = async () => { return await assertExists(await getRelativePathToFile(YAML_SCHEMA_PATH)); }; + +export async function loadAndValidateOpenApiYaml(path: string) { + const yaml = YAML.load(await fs.readFile(path, 'utf8')); + await Parser.validate(cloneDeep(yaml) as any); + return yaml; +} From 87e7ba1a1e3d0e7f2197448852c1c58ebc948919 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Tue, 15 Oct 2024 21:28:58 -0400 Subject: [PATCH 09/20] fix test case Signed-off-by: aramissennyeydd --- .../commands/package/schema/openapi/generate/index.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts index 21a11c6165..73d26924be 100644 --- a/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts +++ b/packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts @@ -52,6 +52,7 @@ describe('generateOpenApiSchema', () => { getPathToCurrentOpenApiSpec: jest.fn(() => Promise.resolve(path.join(inputDir.path, 'openapi.yaml')), ), + loadAndValidateOpenApiYaml: jest.fn(), })); }); it('should handle watch mode', async () => { @@ -89,9 +90,11 @@ describe('generateOpenApiSchema', () => { await new Promise(r => setTimeout(r, 100)); } - expect(generateClientMock).toHaveBeenCalledTimes(0); - mockOn.change(); expect(generateClientMock).toHaveBeenCalledTimes(1); + mockOn.change(); + // Wait for the debounce to finish with the initial load. + await new Promise(r => setTimeout(r, 500)); + expect(generateClientMock).toHaveBeenCalledTimes(2); resolve(); }; From 04297a0c119e259f96a7e6c95773e44678f23840 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 17 Oct 2024 09:55:20 +0200 Subject: [PATCH 10/20] cli: refactor success cache to be additive Signed-off-by: Patrik Oldsberg --- .changeset/loud-trainers-yawn.md | 5 ++ packages/cli/src/commands/repo/lint.ts | 40 ++-------- packages/cli/src/commands/repo/test.ts | 39 ++-------- packages/cli/src/lib/cache/SuccessCache.ts | 89 ++++++++++++++++++++++ 4 files changed, 107 insertions(+), 66 deletions(-) create mode 100644 .changeset/loud-trainers-yawn.md create mode 100644 packages/cli/src/lib/cache/SuccessCache.ts diff --git a/.changeset/loud-trainers-yawn.md b/.changeset/loud-trainers-yawn.md new file mode 100644 index 0000000000..b48f773caa --- /dev/null +++ b/.changeset/loud-trainers-yawn.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The `--successCache` option for the `repo test` and `repo lint` commands now use an additive store that keeps old entries around for a week before they are cleaned up automatically. diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index 1895597e96..9b1ce4ee3e 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -16,9 +16,8 @@ import chalk from 'chalk'; import { Command, OptionValues } from 'commander'; -import fs from 'fs-extra'; import { createHash } from 'crypto'; -import { relative as relativePath, resolve as resolvePath } from 'path'; +import { relative as relativePath } from 'path'; import { PackageGraph, BackstagePackageJson, @@ -27,6 +26,7 @@ import { import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; import { createScriptOptionsParser } from './optionsParser'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; function depCount(pkg: BackstagePackageJson) { const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0; @@ -36,39 +36,13 @@ function depCount(pkg: BackstagePackageJson) { return deps + devDeps; } -const CACHE_FILE_NAME = 'lint-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -async function writeCache(dir: string, cache: Cache) { - await fs.mkdirp(dir); - await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); + const cache = new SuccessCache('lint', opts.successCacheDir); const cacheContext = opts.successCache ? { - cache: await readCache(cacheDir), + entries: await cache.read(), lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), } : undefined; @@ -136,7 +110,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix: Boolean(opts.fix), format: opts.format as string | undefined, shouldCache: Boolean(cacheContext), - successCache: cacheContext?.cache, + successCache: cacheContext?.entries, rootDir: paths.targetRoot, }, workerFactory: async ({ @@ -202,7 +176,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { hash.update('\0'); } sha = await hash.digest('hex'); - if (successCache?.includes(sha)) { + if (successCache?.has(sha)) { console.log(`Skipped ${relativeDir} due to cache hit`); return { relativeDir, sha, failed: false }; } @@ -262,7 +236,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } if (cacheContext) { - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); } if (failed) { diff --git a/packages/cli/src/commands/repo/test.ts b/packages/cli/src/commands/repo/test.ts index 1d59ba3d63..1035354eb2 100644 --- a/packages/cli/src/commands/repo/test.ts +++ b/packages/cli/src/commands/repo/test.ts @@ -16,14 +16,14 @@ import os from 'os'; import crypto from 'node:crypto'; -import fs from 'fs-extra'; import yargs from 'yargs'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { relative as relativePath } from 'path'; import { Command, OptionValues } from 'commander'; import { Lockfile, PackageGraph } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runCheck, runPlain } from '../../lib/run'; import { isChildPath } from '@backstage/cli-common'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; type JestProject = { displayName: string; @@ -50,30 +50,6 @@ interface GlobalWithCache extends Global { }; } -const CACHE_FILE_NAME = 'test-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -function writeCache(dir: string, cache: Cache) { - fs.mkdirpSync(dir); - fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - /** * Use git to get the HEAD tree hashes of each package in the project. */ @@ -272,10 +248,6 @@ export async function command(opts: OptionValues, cmd: Command): Promise { removeOptionArg(args, '--successCache', 1); removeOptionArg(args, '--successCacheDir'); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); - // Parse the args to ensure that no file filters are provided, in which case we refuse to run const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions) .argv; @@ -293,6 +265,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { ); } + const cache = new SuccessCache('test', opts.successCacheDir); const graph = await getPackageGraph(); // Shared state for the bridge @@ -305,7 +278,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { globalWithCache.__backstageCli_jestSuccessCache = { // This is called by `config/jest.js` after the project configs have been gathered async filterConfigs(projectConfigs, globalRootConfig) { - const cache = await readCache(cacheDir); + const cacheEntries = await cache.read(); const lockfile = await Lockfile.load( paths.resolveTargetRoot('yarn.lock'), ); @@ -350,7 +323,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { projectHashes.set(packageName, sha); - if (cache?.includes(sha)) { + if (cacheEntries.has(sha)) { if (!selectedProjects || selectedProjects.includes(packageName)) { console.log(`Skipped ${packageName} due to cache hit`); } @@ -391,7 +364,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } } - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); }, }; } diff --git a/packages/cli/src/lib/cache/SuccessCache.ts b/packages/cli/src/lib/cache/SuccessCache.ts new file mode 100644 index 0000000000..29f4301eea --- /dev/null +++ b/packages/cli/src/lib/cache/SuccessCache.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'node:path'; + +const DEFAULT_CACHE_BASE_PATH = 'node_modules/.cache/backstage-cli'; + +const CACHE_MAX_AGE_MS = 7 * 24 * 3600_000; + +export class SuccessCache { + readonly #path: string; + + constructor(name: string, basePath?: string) { + this.#path = resolvePath(basePath ?? DEFAULT_CACHE_BASE_PATH, name); + } + + async read(): Promise> { + const state = await fs.stat(this.#path).catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (!state || !state.isDirectory()) { + return new Set(); + } + const items = await fs.readdir(this.#path); + + const returned = new Set(); + const removed = new Set(); + + const now = Date.now(); + + for (const item of items) { + const split = item.split('_'); + if (split.length !== 2) { + removed.add(item); + continue; + } + const createdAt = parseInt(split[0], 10); + if (Number.isNaN(createdAt) || now - createdAt > CACHE_MAX_AGE_MS) { + removed.add(item); + } else { + returned.add(split[1]); + } + } + + for (const item of removed) { + await fs.unlink(resolvePath(this.#path, item)); + } + + return returned; + } + + async write(newEntries: Iterable): Promise { + const now = Date.now(); + + await fs.ensureDir(this.#path); + + const existingItems = await fs.readdir(this.#path); + + const empty = Buffer.alloc(0); + for (const key of newEntries) { + // Remove any existing items with the key we're about to add + const trimmedItems = existingItems.filter(item => + item.endsWith(`_${key}`), + ); + for (const trimmedItem of trimmedItems) { + await fs.unlink(resolvePath(this.#path, trimmedItem)); + } + + await fs.writeFile(resolvePath(this.#path, `${now}_${key}`), empty); + } + } +} From e0aafa51ce9c452333a5277f34260fa5c1e8f5bc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 17 Oct 2024 20:53:03 +0200 Subject: [PATCH 11/20] cli: gracefully handle file in place of success cache dir Signed-off-by: Patrik Oldsberg --- packages/cli/src/lib/cache/SuccessCache.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/lib/cache/SuccessCache.ts b/packages/cli/src/lib/cache/SuccessCache.ts index 29f4301eea..5370958537 100644 --- a/packages/cli/src/lib/cache/SuccessCache.ts +++ b/packages/cli/src/lib/cache/SuccessCache.ts @@ -29,15 +29,19 @@ export class SuccessCache { } async read(): Promise> { - const state = await fs.stat(this.#path).catch(error => { + try { + const stat = await fs.stat(this.#path); + if (!stat.isDirectory()) { + await fs.rm(this.#path); + return new Set(); + } + } catch (error) { if (error.code === 'ENOENT') { - return undefined; + return new Set(); } throw error; - }); - if (!state || !state.isDirectory()) { - return new Set(); } + const items = await fs.readdir(this.#path); const returned = new Set(); From 1ff8ca30282a69e5bf29af64a2fbbeec84e86422 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 18 Oct 2024 15:02:22 +0200 Subject: [PATCH 12/20] catalog-client: fix case sensitive filters in mock api Signed-off-by: Patrik Oldsberg --- .changeset/two-donuts-float.md | 5 ++++ .../testUtils/InMemoryCatalogClient.test.ts | 24 +++++++++++++++++++ .../src/testUtils/InMemoryCatalogClient.ts | 19 +++++++++++---- 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 .changeset/two-donuts-float.md diff --git a/.changeset/two-donuts-float.md b/.changeset/two-donuts-float.md new file mode 100644 index 0000000000..d9df082910 --- /dev/null +++ b/.changeset/two-donuts-float.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +Fix for certain filter fields in the `catalogApiMock` being case sensitive. diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts index 3f3eaed477..b77fcd0676 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { CATALOG_FILTER_EXISTS } from '../types'; import { InMemoryCatalogClient } from './InMemoryCatalogClient'; import { Entity } from '@backstage/catalog-model'; @@ -25,6 +26,7 @@ const entity1: Entity = { name: 'e1', uid: 'u1', }, + relations: [{ type: 'relatedTo', targetRef: 'customkind:default/e2' }], }; const entity2: Entity = { @@ -42,10 +44,32 @@ const entities = [entity1, entity2]; describe('InMemoryCatalogClient', () => { it('getEntities', async () => { const client = new InMemoryCatalogClient({ entities }); + await expect(client.getEntities()).resolves.toEqual({ items: entities }); + + await expect( + client.getEntities({ filter: { 'metadata.name': 'E1' } }), + ).resolves.toEqual({ items: [entity1] }); + await expect( client.getEntities({ filter: { 'metadata.uid': 'u2' } }), ).resolves.toEqual({ items: [entity2] }); + + await expect( + client.getEntities({ filter: { 'metadata.uid': 'U2' } }), + ).resolves.toEqual({ items: [entity2] }); + + await expect( + client.getEntities({ + filter: { 'relations.relatedto': CATALOG_FILTER_EXISTS }, + }), + ).resolves.toEqual({ items: [entity1] }); + + await expect( + client.getEntities({ + filter: { 'relations.relatedTo': 'customkind:default/e2' }, + }), + ).resolves.toEqual({ items: [entity1] }); }); it('getEntitiesByRefs', async () => { diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts index 355a10bf0e..13ce190e88 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts @@ -48,13 +48,22 @@ function buildEntitySearch(entity: Entity) { const rows = traverse(entity); if (entity.metadata?.name) { - rows.push({ key: 'metadata.name', value: entity.metadata.name }); + rows.push({ + key: 'metadata.name', + value: entity.metadata.name.toLocaleLowerCase('en-US'), + }); } if (entity.metadata?.namespace) { - rows.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); + rows.push({ + key: 'metadata.namespace', + value: entity.metadata.namespace.toLocaleLowerCase('en-US'), + }); } if (entity.metadata?.uid) { - rows.push({ key: 'metadata.uid', value: entity.metadata.uid }); + rows.push({ + key: 'metadata.uid', + value: entity.metadata.uid.toLocaleLowerCase('en-US'), + }); } if (!entity.metadata.namespace) { @@ -64,8 +73,8 @@ function buildEntitySearch(entity: Entity) { // Visit relations for (const relation of entity.relations ?? []) { rows.push({ - key: `relations.${relation.type}`, - value: relation.targetRef, + key: `relations.${relation.type.toLocaleLowerCase('en-US')}`, + value: relation.targetRef.toLocaleLowerCase('en-US'), }); } From 41c717596bcaad74f229744e68451788fb706fdb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 19 Oct 2024 07:59:37 +0000 Subject: [PATCH 13/20] chore(deps): update dependency @types/lodash to v4.17.12 Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 448a5bffef..ac0a0187d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18122,9 +18122,9 @@ __metadata: linkType: hard "@types/lodash@npm:^4.14.151": - version: 4.17.10 - resolution: "@types/lodash@npm:4.17.10" - checksum: 4600f2f25270c8fee6953e363d318149a5f0f1b1bb820aa2f42d7ada6e4f7de31848bb5ffc2c687b40bd73aa982167bdd6e6d8d456e72abe0c660ec77d1fa7e9 + version: 4.17.12 + resolution: "@types/lodash@npm:4.17.12" + checksum: 7b564e4114f09ce5ae31a2e9493592baf20bb498507f3705c5d91cf838c2298b4f6a06f2d6c8dc608fcac63e210a2b7b13388c7a5e220e15688f813521030127 languageName: node linkType: hard From c49926d3abd6d3f3c882ca2768fc335c4ec4140b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 19 Oct 2024 08:51:22 +0000 Subject: [PATCH 14/20] chore(deps): update dependency @types/node to v18.19.57 Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index ac0a0187d5..746f0e66d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18289,11 +18289,11 @@ __metadata: linkType: hard "@types/node@npm:*, @types/node@npm:>=13.7.0": - version: 22.7.6 - resolution: "@types/node@npm:22.7.6" + version: 22.7.7 + resolution: "@types/node@npm:22.7.7" dependencies: undici-types: ~6.19.2 - checksum: 6afe2a1bd70ee0afa76c904ffdb04c634366987103bb2bef2d5b036f3bb49a4e828bb5e9fc64af442a76aaac4896df9277e725c843d85375d135aa71eb2fe3d9 + checksum: 70492e46d92bf00b537c8700322ad001c1f8d4fc65fc1627064a91c7edebb8ad18730b95a3c6bdff212dd0252a337a4b92e4cfcbc21c9f08616302e87ea855b8 languageName: node linkType: hard @@ -18326,20 +18326,20 @@ __metadata: linkType: hard "@types/node@npm:^18.11.18, @types/node@npm:^18.11.9, @types/node@npm:^18.17.15, @types/node@npm:^18.17.8": - version: 18.19.56 - resolution: "@types/node@npm:18.19.56" + version: 18.19.57 + resolution: "@types/node@npm:18.19.57" dependencies: undici-types: ~5.26.4 - checksum: 2660aa2e50ddb77e4ea4e29e6b917e942ca49ecbd46b6333a7aeffdffafc5d7e48f8a731b6570a61bebdd0ba8dc104a250c432bea00e70aec5a4cb6e6f2df771 + checksum: d7be072513df5f6b14384dc284561ddf4e6393ce85b8fd3cc9ab0c0a7abd03bfdab6519548dab7c743ec9dfca9ce686dde2db0fbbf1d357e0822b4f99a54b193 languageName: node linkType: hard "@types/node@npm:^20.1.1, @types/node@npm:^20.11.16": - version: 20.16.12 - resolution: "@types/node@npm:20.16.12" + version: 20.16.13 + resolution: "@types/node@npm:20.16.13" dependencies: undici-types: ~6.19.2 - checksum: 648b2a35713157f1d5861123c29546cf316b543afd536ff2c02234d78a99cea07d2259559efd7f58cf2c45c6a2e80c1064ff03c5b5213bb99b91a7d598b7975f + checksum: 2e998ac5ae133b1a4261dc89d0034e63834d285bb2124901b9d805e7b98c0f809fc2586fd1bf7ad9a0f0348bb503fac6c41612a3ca272334b943070a3e82227c languageName: node linkType: hard From 28b60adbbe6111f85b4307df6347784d3d133684 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 19 Oct 2024 11:26:27 +0200 Subject: [PATCH 15/20] cli: fix react-dom/client check in Jest config Signed-off-by: Patrik Oldsberg --- .changeset/chilled-taxis-dance.md | 5 +++++ packages/cli/config/jest.js | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changeset/chilled-taxis-dance.md diff --git a/.changeset/chilled-taxis-dance.md b/.changeset/chilled-taxis-dance.md new file mode 100644 index 0000000000..e2a5dcecee --- /dev/null +++ b/.changeset/chilled-taxis-dance.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The check for `react-dom/client` in the Jest configuration will now properly always run from the target directory. diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index 36ffd359aa..b979db5177 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -28,7 +28,9 @@ const envOptions = { }; try { - require.resolve('react-dom/client'); + require.resolve('react-dom/client', { + paths: [paths.targetRoot], + }); process.env.HAS_REACT_DOM_CLIENT = true; } catch { /* ignored */ From 3f109966b47ad83eaf8fcabae02db826502dae0b Mon Sep 17 00:00:00 2001 From: Peter Macdonald Date: Sat, 19 Oct 2024 16:43:40 +0200 Subject: [PATCH 16/20] Ads some additional notes to the coreservices docs to just nudge people in the right direction Signed-off-by: Peter Macdonald --- docs/backend-system/core-services/identity.md | 6 ++++++ docs/backend-system/core-services/user-info.md | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/backend-system/core-services/identity.md b/docs/backend-system/core-services/identity.md index eb7d1f4ac7..b4baf99682 100644 --- a/docs/backend-system/core-services/identity.md +++ b/docs/backend-system/core-services/identity.md @@ -5,6 +5,12 @@ sidebar_label: Identity description: Documentation for the Identity service --- +:::note Note + +This service is deprecated, you are likely looking for the [Auth Service](./auth.md) instead. If you're wondering how to get the user's entity ref and ownership claims in your backend plugin, you should see the [User Info Service](./user-info.md) documentation. + +::: + When working with backend plugins, you might find that you will need to interact with the `auth-backend` plugin to both authenticate backstage tokens, and to deconstruct them to get the user's entity ref and/or ownership claims out of them. ## Using the service diff --git a/docs/backend-system/core-services/user-info.md b/docs/backend-system/core-services/user-info.md index 73f461d1ec..3a2480dd9f 100644 --- a/docs/backend-system/core-services/user-info.md +++ b/docs/backend-system/core-services/user-info.md @@ -9,8 +9,12 @@ This service lets you extract more information about a set of user credentials. Specifically, it can be used to extract the ownership entity refs for a user principal. -See also the [`auth`](./auth.md) and [`httpAuth`](./http-auth.md) services for -general credentials handling. +:::note Note + +Please also refer to [`auth`](./auth.md) and [`httpAuth`](./http-auth.md) services for +general credentials handling which is a prerequisite for the below examples. + +::: ## Using the Service From e30b65de7ede97884f61eb78c647284a0551a695 Mon Sep 17 00:00:00 2001 From: Tyler Davis Date: Sun, 20 Oct 2024 14:09:52 +1100 Subject: [PATCH 17/20] replace --alwaysYarnPack with --alwaysPack Signed-off-by: Tyler Davis --- .changeset/khaki-fireants-march.md | 5 +++++ packages/cli/cli-report.md | 2 +- packages/cli/src/commands/buildWorkspace.ts | 4 ++-- packages/cli/src/commands/index.ts | 12 ++++++++++-- packages/cli/src/lib/packager/createDistWorkspace.ts | 8 ++++---- 5 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 .changeset/khaki-fireants-march.md diff --git a/.changeset/khaki-fireants-march.md b/.changeset/khaki-fireants-march.md new file mode 100644 index 0000000000..600f8eba9e --- /dev/null +++ b/.changeset/khaki-fireants-march.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +add --alwaysPack as a replacement for the now-hidden --alwaysYarnPack diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index f164e5e1fc..e17af5a312 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -34,7 +34,7 @@ Commands: Usage: backstage-cli build-workspace [options] [packages...] Options: - --alwaysYarnPack + --alwaysPack -h, --help ``` diff --git a/packages/cli/src/commands/buildWorkspace.ts b/packages/cli/src/commands/buildWorkspace.ts index a20663fc32..159c347eb8 100644 --- a/packages/cli/src/commands/buildWorkspace.ts +++ b/packages/cli/src/commands/buildWorkspace.ts @@ -18,7 +18,7 @@ import fs from 'fs-extra'; import { createDistWorkspace } from '../lib/packager'; type Options = { - alwaysYarnPack?: boolean; + alwaysPack?: boolean; }; export default async (dir: string, packages: string[], options: Options) => { @@ -28,7 +28,7 @@ export default async (dir: string, packages: string[], options: Options) => { await createDistWorkspace(packages, { targetDir: dir, - alwaysYarnPack: options.alwaysYarnPack, + alwaysPack: options.alwaysPack, enableFeatureDetection: true, }); }; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index a3060f2d67..369138cb1a 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Command } from 'commander'; +import { Command, Option } from 'commander'; import { assertError } from '@backstage/errors'; import { exitWithError } from '../lib/errors'; @@ -375,8 +375,16 @@ export function registerCommands(program: Command) { program .command('build-workspace [packages...]') + .addOption( + new Option( + '--alwaysYarnPack', + 'Alias for --alwaysPack for backwards compatibility.', + ) + .implies({ alwaysPack: true }) + .hideHelp(true), + ) .option( - '--alwaysYarnPack', + '--alwaysPack', 'Force workspace output to be a result of running `yarn pack` on each package (warning: very slow)', ) .description('Builds a temporary dist workspace from the provided packages') diff --git a/packages/cli/src/lib/packager/createDistWorkspace.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts index 2363023ddd..ea1682bc4f 100644 --- a/packages/cli/src/lib/packager/createDistWorkspace.ts +++ b/packages/cli/src/lib/packager/createDistWorkspace.ts @@ -103,7 +103,7 @@ type Options = { * workspace. This ensures correct workspace output at significant cost to * command performance. */ - alwaysYarnPack?: boolean; + alwaysPack?: boolean; /** * If set to true, the TypeScript feature detection will be enabled, which @@ -242,7 +242,7 @@ export async function createDistWorkspace( await moveToDistWorkspace( targetDir, targets, - Boolean(options.alwaysYarnPack), + Boolean(options.alwaysPack), Boolean(options.enableFeatureDetection), ); @@ -286,13 +286,13 @@ const FAST_PACK_SCRIPTS = [ async function moveToDistWorkspace( workspaceDir: string, localPackages: PackageGraphNode[], - alwaysYarnPack: boolean, + alwaysPack: boolean, enableFeatureDetection: boolean, ): Promise { const [fastPackPackages, slowPackPackages] = partition( localPackages, pkg => - !alwaysYarnPack && + !alwaysPack && FAST_PACK_SCRIPTS.includes(pkg.packageJson.scripts?.prepack), ); From 106c3c40b46d038a7b7498edaae416fa5c5aa8db Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 20 Oct 2024 11:08:04 +0200 Subject: [PATCH 18/20] Apply suggestions from code review Signed-off-by: Patrik Oldsberg --- .changeset/khaki-fireants-march.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/khaki-fireants-march.md b/.changeset/khaki-fireants-march.md index 600f8eba9e..8b5d897305 100644 --- a/.changeset/khaki-fireants-march.md +++ b/.changeset/khaki-fireants-march.md @@ -2,4 +2,4 @@ '@backstage/cli': patch --- -add --alwaysPack as a replacement for the now-hidden --alwaysYarnPack +Added `--alwaysPack` as a replacement for the now hidden `--alwaysYarnPack` option for the `build-workspace` command. From 8733d576bfaddfe70cb84bf5cbdccdc5da6b03c7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 20 Oct 2024 11:08:30 +0200 Subject: [PATCH 19/20] Apply suggestions from code review Signed-off-by: Patrik Oldsberg --- .changeset/khaki-fireants-march.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/khaki-fireants-march.md b/.changeset/khaki-fireants-march.md index 8b5d897305..ec84fe1b63 100644 --- a/.changeset/khaki-fireants-march.md +++ b/.changeset/khaki-fireants-march.md @@ -2,4 +2,4 @@ '@backstage/cli': patch --- -Added `--alwaysPack` as a replacement for the now hidden `--alwaysYarnPack` option for the `build-workspace` command. +Added `--alwaysPack` as a replacement for the now hidden `--alwaysYarnPack` flag for the `build-workspace` command. From eb829944a2c835c8e65ccd9318bee7c10d8a2757 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 20 Oct 2024 10:24:27 +0200 Subject: [PATCH 20/20] backend-test-utils: remove unused msw dep Signed-off-by: Patrik Oldsberg --- .changeset/shy-pots-speak.md | 5 +++++ packages/backend-test-utils/package.json | 1 - yarn.lock | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .changeset/shy-pots-speak.md diff --git a/.changeset/shy-pots-speak.md b/.changeset/shy-pots-speak.md new file mode 100644 index 0000000000..8eceacb9ec --- /dev/null +++ b/.changeset/shy-pots-speak.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +Removed unused `msw` dependency. diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index 9d73fdd485..75136b6169 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -65,7 +65,6 @@ "fs-extra": "^11.0.0", "keyv": "^4.5.2", "knex": "^3.0.0", - "msw": "^1.0.0", "mysql2": "^3.0.0", "pg": "^8.11.3", "pg-connection-string": "^2.3.0", diff --git a/yarn.lock b/yarn.lock index 746f0e66d5..b2fadb108e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3809,7 +3809,6 @@ __metadata: fs-extra: ^11.0.0 keyv: ^4.5.2 knex: ^3.0.0 - msw: ^1.0.0 mysql2: ^3.0.0 pg: ^8.11.3 pg-connection-string: ^2.3.0