Merge pull request #25539 from aramissennyeydd/openapi-tooling/watch

feat(openapi-tooling): add a watch mode for generate command

.changeset/slow-mayflies-sleep.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': minor
+---
+
+Adds a `--watch` mode to the `schema openapi generate` command for a better local schema writing experience.

packages/repo-tools/package.json

@@ -62,6 +62,7 @@
     "@stoplight/types": "^14.0.0",
     "@useoptic/openapi-utilities": "^0.55.0",
     "chalk": "^4.0.0",
+    "chokidar": "^3.5.3",
     "codeowners-utils": "^1.0.2",
     "command-exists": "^1.2.9",
     "commander": "^12.0.0",

packages/repo-tools/src/commands/index.ts

@@ -58,6 +58,8 @@ function registerPackageCommand(program: Command) {
     .description(
       'Additional properties that can be passed to @openapitools/openapi-generator-cli',
     )
+    .option('--watch')
+    .description('Watch the OpenAPI spec for changes and regenerate on save.')
     .action(
       lazy(() =>
         import('./package/schema/openapi/generate').then(m => m.command),

packages/repo-tools/src/commands/package/schema/openapi/generate/client.ts

@@ -30,6 +30,7 @@ import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'
 async function generate(
   outputDirectory: string,
   clientAdditionalProperties?: string,
+  abortSignal?: AbortController,
 ) {
   const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
   const resolvedOutputDirectory = cliPaths.resolveTargetRoot(
@@ -69,6 +70,7 @@ async function generate(
       additionalProperties,
     ],
     {
+      signal: abortSignal?.signal,
       maxBuffer: Number.MAX_VALUE,
       cwd: resolvePackagePath('@backstage/repo-tools'),
       env: {
@@ -79,11 +81,15 @@ async function generate(
 
   await exec(
     `yarn backstage-cli package lint --fix ${resolvedOutputDirectory}`,
+    [],
+    { signal: abortSignal?.signal },
   );
 
   const prettier = cliPaths.resolveTargetRoot('node_modules/.bin/prettier');
   if (prettier) {
-    await exec(`${prettier} --write ${resolvedOutputDirectory}`);
+    await exec(`${prettier} --write ${resolvedOutputDirectory}`, [], {
+      signal: abortSignal?.signal,
+    });
   }
 
   fs.removeSync(resolve(resolvedOutputDirectory, '.openapi-generator-ignore'));
@@ -97,17 +103,29 @@ async function generate(
 export async function command(
   outputPackage: string,
   clientAdditionalProperties?: string,
+  {
+    abortSignal,
+    isWatch = false,
+  }: { abortSignal?: AbortController; isWatch?: boolean } = {},
 ): Promise<void> {
   try {
-    await generate(outputPackage, clientAdditionalProperties);
+    await generate(outputPackage, clientAdditionalProperties, abortSignal);
     console.log(
       chalk.green(`Generated client in ${outputPackage}/${OUTPUT_PATH}`),
     );
   } catch (err) {
-    console.log();
-    console.log(chalk.red(`Client generation failed:`));
-    console.log(err);
-
-    process.exit(1);
+    if (err.name === 'AbortError') {
+      console.debug('Server generation aborted.');
+      return;
+    }
+    if (isWatch) {
+      console.log(chalk.red(`Client generation failed:`));
+      console.group();
+      console.log(chalk.red(err.message));
+      console.groupEnd();
+    } else {
+      console.log(chalk.red(`Client generation failed.`));
+      console.log(chalk.red(err.message));
+    }
   }
 }

packages/repo-tools/src/commands/package/schema/openapi/generate/index.test.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import path from 'path';
+
+jest.mock(
+  'lodash',
+  () =>
+    ({
+      ...jest.requireActual('lodash'),
+      debounce: (fn: any) => fn,
+    } as any as typeof import('lodash')),
+);
+
+describe('generateOpenApiSchema', () => {
+  const inputDir = createMockDirectory();
+  const outputDir = createMockDirectory();
+
+  beforeEach(() => {
+    inputDir.clear();
+    outputDir.clear();
+
+    inputDir.addContent({
+      'openapi.yaml': `
+          openapi: 3.0.0
+          info:
+            title: Test API
+            version: 1.0.0
+          paths:
+            /test:
+              get:
+                responses:
+                  '200':
+                    description: OK
+        `,
+    });
+    jest.mock('../../../../../lib/openapi/helpers', () => ({
+      getPathToCurrentOpenApiSpec: jest.fn(() =>
+        Promise.resolve(path.join(inputDir.path, 'openapi.yaml')),
+      ),
+      loadAndValidateOpenApiYaml: jest.fn(),
+    }));
+  });
+  it('should handle watch mode', async () => {
+    const generateClientMock = jest.fn();
+    const generateServerMock = jest.fn();
+    jest.doMock('./client', () => ({ command: generateClientMock }));
+    jest.doMock('./server', () => ({ command: generateServerMock }));
+
+    const mockWatch = jest.fn();
+    jest.mock('chokidar', () => ({
+      watch: mockWatch,
+    }));
+    let resolve: (val?: any) => void;
+    const block = () =>
+      new Promise(res => {
+        resolve = res;
+      });
+    jest.mock('../../../../../lib/runner', () => ({
+      block,
+    }));
+
+    const mockOn: Record<string, any> = {};
+    mockWatch.mockReturnValue({
+      on: jest.fn((event, cb) => {
+        console.log(event);
+        mockOn[event] = cb;
+      }),
+    } as any);
+
+    const { command } = await import('./index');
+
+    const actions = async () => {
+      while (!mockOn.ready) {
+        // Wait for the watcher to be registered
+        await new Promise(r => setTimeout(r, 100));
+      }
+
+      expect(generateClientMock).toHaveBeenCalledTimes(1);
+      mockOn.change();
+      // Wait for the debounce to finish with the initial load.
+      await new Promise(r => setTimeout(r, 500));
+      expect(generateClientMock).toHaveBeenCalledTimes(2);
+      resolve();
+    };
+
+    await Promise.all([
+      actions(),
+      command({ watch: true, clientPackage: 'test123' }),
+    ]);
+  });
+});

packages/repo-tools/src/commands/package/schema/openapi/generate/index.ts

@@ -17,6 +17,13 @@ import chalk from 'chalk';
 import { OptionValues } from 'commander';
 import { command as generateClient } from './client';
 import { command as generateServer } from './server';
+import chokidar from 'chokidar';
+import {
+  getPathToCurrentOpenApiSpec,
+  loadAndValidateOpenApiYaml,
+} from '../../../../../lib/openapi/helpers';
+import { debounce } from 'lodash';
+import { block } from '../../../../../lib/runner';
 
 export async function command(opts: OptionValues) {
   if (!opts.clientPackage && !opts.server) {
@@ -25,10 +32,70 @@ export async function command(opts: OptionValues) {
     );
     process.exit(1);
   }
-  if (opts.clientPackage) {
-    await generateClient(opts.clientPackage, opts.clientAdditionalProperties);
-  }
-  if (opts.server) {
-    await generateServer();
+
+  const sharedCommand = async (abortSignal?: AbortController) => {
+    const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+    await loadAndValidateOpenApiYaml(resolvedOpenapiPath);
+    const promises = [];
+    const options = {
+      isWatch: opts.watch,
+      abortSignal,
+    };
+    if (opts.clientPackage) {
+      promises.push(
+        generateClient(
+          opts.clientPackage,
+          opts.clientAdditionalProperties,
+          options,
+        ),
+      );
+    }
+    if (opts.server) {
+      promises.push(generateServer(options));
+    }
+    await Promise.all(promises);
+  };
+
+  if (opts.watch) {
+    try {
+      const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+      let abortController = new AbortController();
+      const watcher = chokidar.watch(resolvedOpenapiPath);
+
+      // The generate command currently takes ~8 seconds to run, so let's debounce calling it so we don't have to cancel it so much.
+      const debouncedCommand = debounce(() => {
+        console.log('Detected changes! Regenerating...');
+        abortController.abort();
+        abortController = new AbortController();
+        sharedCommand(abortController).catch(err => {
+          console.error(chalk.red('Error: ', err));
+        });
+      }, 500);
+
+      watcher.on('change', () => {
+        debouncedCommand();
+      });
+      watcher.on('error', error => {
+        console.error('Error happened', error);
+      });
+
+      watcher.on('ready', async () => {
+        console.log(
+          'Watching for changes in OpenAPI spec. Press Ctrl+C to stop.',
+        );
+      });
+
+      debouncedCommand();
+      await block();
+    } catch (err) {
+      console.error(chalk.red('Error: ', err));
+      process.exit(1);
+    }
+  } else {
+    try {
+      await sharedCommand();
+    } catch (err) {
+      process.exit(1);
+    }
   }
 }

packages/repo-tools/src/commands/package/schema/openapi/generate/server.ts

@@ -25,7 +25,7 @@ import { getPathToCurrentOpenApiSpec } from '../../../../../lib/openapi/helpers'
 
 const exec = promisify(execCb);
 
-async function generate() {
+async function generate(abortSignal?: AbortController) {
   const openapiPath = await getPathToCurrentOpenApiSpec();
   const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8'));
 
@@ -50,21 +50,40 @@ export const createOpenApiRouter = async (
 `,
   );
 
-  await exec(`yarn backstage-cli package lint --fix ${tsPath}`);
+  await exec(`yarn backstage-cli package lint --fix ${tsPath}`, {
+    signal: abortSignal?.signal,
+  });
   if (await cliPaths.resolveTargetRoot('node_modules/.bin/prettier')) {
     await exec(`yarn prettier --write ${tsPath}`, {
       cwd: cliPaths.targetRoot,
+      signal: abortSignal?.signal,
     });
   }
 }
 
-export async function command(): Promise<void> {
+export async function command({
+  abortSignal,
+  isWatch = false,
+}: {
+  abortSignal?: AbortController;
+  isWatch?: boolean;
+}): Promise<void> {
   try {
-    await generate();
-    console.log(chalk.green('Generated all files.'));
+    await generate(abortSignal);
+    console.log(chalk.green('Generated server files.'));
   } catch (err) {
-    console.log(chalk.red(`OpenAPI server stub generation failed.`));
-    console.log(err.message);
-    process.exit(1);
+    if (err.name === 'AbortError') {
+      console.debug('Server generation aborted.');
+      return;
+    }
+    if (isWatch) {
+      console.log(chalk.red(`Server generation failed:`));
+      console.group();
+      console.log(chalk.red(err.message));
+      console.groupEnd();
+    } else {
+      console.log(chalk.red(err.message));
+      console.log(chalk.red(`OpenAPI server stub generation failed.`));
+    }
   }
 }

packages/repo-tools/src/commands/repo/schema/openapi/verify.ts

@@ -15,12 +15,10 @@
  */
 
 import fs from 'fs-extra';
-import YAML from 'js-yaml';
-import { isEqual, cloneDeep } from 'lodash';
+import { isEqual } from 'lodash';
 import { join } from 'path';
 import chalk from 'chalk';
 import { relative as relativePath, resolve as resolvePath } from 'path';
-import Parser from '@apidevtools/swagger-parser';
 import { runner } from '../../../../lib/runner';
 import { paths as cliPaths } from '../../../../lib/paths';
 import {
@@ -28,7 +26,10 @@ import {
   TS_SCHEMA_PATH,
   YAML_SCHEMA_PATH,
 } from '../../../../lib/openapi/constants';
-import { getPathToOpenApiSpec } from '../../../../lib/openapi/helpers';
+import {
+  getPathToOpenApiSpec,
+  loadAndValidateOpenApiYaml,
+} from '../../../../lib/openapi/helpers';
 
 async function verify(directoryPath: string) {
   let openapiPath = '';
@@ -38,8 +39,7 @@ async function verify(directoryPath: string) {
     // Unable to find spec at path.
     return;
   }
-  const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8'));
-  await Parser.validate(cloneDeep(yaml) as any);
+  const yaml = await loadAndValidateOpenApiYaml(openapiPath);
 
   const schemaPath = join(directoryPath, TS_SCHEMA_PATH);
   if (!(await fs.pathExists(schemaPath))) {

packages/repo-tools/src/lib/openapi/helpers.ts

@@ -18,6 +18,10 @@ import { pathExists } from 'fs-extra';
 import { paths } from '../paths';
 import { YAML_SCHEMA_PATH } from './constants';
 import { resolve } from 'path';
+import YAML from 'js-yaml';
+import { cloneDeep } from 'lodash';
+import Parser from '@apidevtools/swagger-parser';
+import fs from 'fs-extra';
 
 export const getPathToFile = async (directory: string, filename: string) => {
   return resolve(directory, filename);
@@ -41,3 +45,9 @@ export const getPathToOpenApiSpec = async (directory: string) => {
 export const getPathToCurrentOpenApiSpec = async () => {
   return await assertExists(await getRelativePathToFile(YAML_SCHEMA_PATH));
 };
+
+export async function loadAndValidateOpenApiYaml(path: string) {
+  const yaml = YAML.load(await fs.readFile(path, 'utf8'));
+  await Parser.validate(cloneDeep(yaml) as any);
+  return yaml;
+}

packages/repo-tools/src/lib/runner.ts

@@ -67,3 +67,7 @@ export async function runner(
 
   return resultsList;
 }
+
+export async function block() {
+  return new Promise(() => {});
+}

yarn.lock

@@ -8310,6 +8310,7 @@ __metadata:
     "@types/prettier": ^2.0.0
     "@useoptic/openapi-utilities": ^0.55.0
     chalk: ^4.0.0
+    chokidar: ^3.5.3
     codeowners-utils: ^1.0.2
     command-exists: ^1.2.9
     commander: ^12.0.0