feat(openapi-tooling): add support for fuzzing with schemathesis

Signed-off-by: aramissennyeydd <aramis.sennyey@doordash.com>

.gitignore

@@ -166,3 +166,7 @@ plugins-report.csv
 
 # Temporary knip configs
 knip.json
+
+# Schemathesis temporary files
+.hypothesis/
+.cassettes/

packages/repo-tools/package.json

@@ -36,6 +36,7 @@
     "@backstage/catalog-model": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/cli-node": "workspace:^",
+    "@backstage/config-loader": "workspace:^",
     "@backstage/errors": "workspace:^",
     "@manypkg/get-packages": "^1.1.3",
     "@microsoft/api-documenter": "^7.22.33",
@@ -50,6 +51,7 @@
     "@stoplight/types": "^14.0.0",
     "chalk": "^4.0.0",
     "codeowners-utils": "^1.0.2",
+    "command-exists": "^1.2.9",
     "commander": "^12.0.0",
     "fs-extra": "^11.2.0",
     "glob": "^8.0.3",

packages/repo-tools/src/commands/index.ts

@@ -59,6 +59,22 @@ function registerPackageCommand(program: Command) {
         import('./package/schema/openapi/generate').then(m => m.command),
       ),
     );
+
+  openApiCommand
+    .command('fuzz')
+    .description(
+      'Fuzz an OpenAPI schema by generating random data and sending it to the server.',
+    )
+    .option('--count <count>', 'Number of requests to send')
+    .option('--workers <workers>', 'Number of workers to use', '2')
+    .option('--debug', 'Enable debug mode')
+    .option(
+      '--exclude-checks <excludeChecks>',
+      'Exclude checks from schemathesis run',
+    )
+    .action(
+      lazy(() => import('./package/schema/openapi/fuzz').then(m => m.command)),
+    );
 }
 
 function registerRepoCommand(program: Command) {
@@ -103,6 +119,17 @@ function registerRepoCommand(program: Command) {
     .action(
       lazy(() => import('./repo/schema/openapi/test').then(m => m.bulkCommand)),
     );
+
+  openApiCommand
+    .command('fuzz')
+    .description('Fuzz all packages')
+    .option(
+      '--since <ref>',
+      'Only fuzz packages that have changed since the given ref',
+    )
+    .action(
+      lazy(() => import('./repo/schema/openapi/fuzz').then(m => m.command)),
+    );
 }
 
 export function registerCommands(program: Command) {

packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import { paths as cliPaths } from '../../../../lib/paths';
+import chalk from 'chalk';
+import { spawn } from '../../../../lib/exec';
+import { getPathToCurrentOpenApiSpec } from '../../../../lib/openapi/helpers';
+import { ConfigSources } from '@backstage/config-loader';
+import YAML from 'js-yaml';
+import { join } from 'path';
+import { OptionValues } from 'commander';
+import { sync as existsSync } from 'command-exists';
+
+async function fuzz(opts: OptionValues) {
+  const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+
+  if (!existsSync('st')) {
+    console.log(
+      chalk.red(
+        `Please install schemathesis globally with 'python -m pip install schemathesis'. Then run this command again.`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  const openapiSpec = YAML.load(
+    await fs.readFile(resolvedOpenapiPath, 'utf8'),
+  ) as { info: { title: string } };
+  const configSource = ConfigSources.default({
+    rootDir: cliPaths.targetRoot,
+  });
+  const config = await ConfigSources.toConfig(configSource);
+  const pluginId = openapiSpec.info.title;
+  const args = [];
+  if (opts.debug) {
+    args.push(
+      '--cassette-path',
+      cliPaths.resolveTargetRoot(join('.cassettes', `${pluginId}`)),
+    );
+  }
+
+  if (opts.count) {
+    args.push('--hypothesis-max-examples', opts.count);
+  }
+  args.push('--workers', opts.workers);
+
+  if (opts.useGuest) {
+    args.push('--header', `Authorization: Basic guest`);
+  } else {
+    args.push('--header', `Authorization: Basic test`);
+  }
+
+  if (opts.excludeChecks) {
+    args.push('--exclude-checks', opts.excludeChecks);
+  }
+
+  await spawn(
+    'st',
+    [
+      'run',
+      '--checks',
+      'all',
+      '--data-generation-method',
+      'all',
+      ...args,
+      `${config.getString('backend.baseUrl')}/api/${pluginId}/openapi.json`,
+    ],
+    {
+      stdio: ['ignore', 'inherit', 'inherit'],
+    },
+  );
+}
+
+export async function command(opts: OptionValues) {
+  try {
+    await fuzz(opts);
+    console.log(chalk.green(`Successfully fuzzed.`));
+  } catch (err) {
+    console.log(chalk.red(`OpenAPI fuzzing failed.`));
+    console.error(err);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PackageGraph } from '@backstage/cli-node';
+import { OptionValues } from 'commander';
+import { exec } from '../../../../lib/exec';
+import chalk from 'chalk';
+
+export async function command(opts: OptionValues) {
+  let packages = await PackageGraph.listTargetPackages();
+  if (opts.since) {
+    const graph = PackageGraph.fromPackages(packages);
+    const changedPackages = await graph.listChangedPackages({
+      ref: opts.since,
+      analyzeLockfile: true,
+    });
+    const withDevDependents = graph.collectPackageNames(
+      changedPackages.map(pkg => pkg.name),
+      pkg => pkg.localDevDependents.keys(),
+    );
+    packages = Array.from(withDevDependents).map(name => graph.get(name)!);
+  }
+
+  const fuzzablePackages = packages.filter(e => e.packageJson.scripts?.fuzz);
+  try {
+    for (const pkg of fuzzablePackages) {
+      await exec('yarn', ['fuzz'], {
+        cwd: pkg.dir,
+      });
+    }
+    console.log(chalk.green(`Successfully fuzzed.`));
+  } catch (err) {
+    console.error(err.stdout);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/lib/exec.ts

@@ -14,20 +14,50 @@
  * limitations under the License.
  */
 import { promisify } from 'util';
-import { ExecOptions, exec as execCb } from 'child_process';
+import {
+  ExecOptions,
+  SpawnOptions,
+  exec as execCb,
+  spawn as spawnOriginal,
+} from 'child_process';
 
 const execPromise = promisify(execCb);
 
 export const exec = (
   command: string,
-  options: string[] = [],
-  execOptions?: ExecOptions,
+  args: string[] = [],
+  options?: ExecOptions,
 ) => {
   return execPromise(
     [
       command,
-      ...options.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)),
+      ...args.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)),
     ].join(' '),
-    execOptions,
+    options,
   );
 };
+
+export const spawn = (
+  command: string,
+  args: string[],
+  options?: SpawnOptions,
+) => {
+  return new Promise((resolve, reject) => {
+    const cp = spawnOriginal(command, args, options ?? {});
+    const error: string[] = [];
+    const stdout: string[] = [];
+
+    cp.stdout?.on('data', data => {
+      stdout.push(data.toString());
+    });
+
+    cp.on('error', e => {
+      error.push(e.toString());
+    });
+
+    cp.on('close', exitCode => {
+      if (exitCode) reject(error.join(''));
+      else resolve(stdout.join(''));
+    });
+  });
+};

plugins/catalog-backend/package.json

@@ -43,6 +43,7 @@
   "scripts": {
     "build": "backstage-cli package build",
     "clean": "backstage-cli package clean",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz --exclude-checks content_type_conformance,response_schema_conformance",
     "generate": "backstage-repo-tools package schema openapi generate --server --client-package packages/catalog-client",
     "lint": "backstage-cli package lint",
     "prepack": "backstage-cli package prepack",

plugins/catalog-backend/src/service/createRouter.ts

@@ -298,7 +298,8 @@ export async function createRouter(
         location: locationInput,
         catalogFilename: z.string().optional(),
       });
-      const output = await locationAnalyzer.analyzeLocation(schema.parse(body));
+      const parsedBody = schema.parse(body);
+      const output = await locationAnalyzer.analyzeLocation(parsedBody);
       res.status(200).json(output);
     });
   }

plugins/catalog-backend/src/service/util.ts

@@ -50,7 +50,7 @@ export async function requireRequestBody(req: Request): Promise<unknown> {
 export const locationInput = z
   .object({
     type: z.string(),
-    target: z.string(),
+    target: z.string().url(),
     presence: z.literal('required').or(z.literal('optional')).optional(),
   })
   .strict(); // no unknown keys;

plugins/search-backend/package.json

@@ -40,7 +40,8 @@
     "prepack": "backstage-cli package prepack",
     "postpack": "backstage-cli package postpack",
     "clean": "backstage-cli package clean",
-    "generate": "backstage-repo-tools package schema openapi generate --server"
+    "generate": "backstage-repo-tools package schema openapi generate --server",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz"
   },
   "dependencies": {
     "@backstage/backend-common": "workspace:^",
@@ -52,6 +53,7 @@
     "@backstage/plugin-permission-node": "workspace:^",
     "@backstage/plugin-search-backend-node": "workspace:^",
     "@backstage/plugin-search-common": "workspace:^",
+    "@backstage/repo-tools": "workspace:^",
     "@backstage/types": "workspace:^",
     "@types/express": "^4.17.6",
     "dataloader": "^2.0.0",

plugins/todo-backend/package.json

@@ -25,6 +25,7 @@
   "scripts": {
     "build": "backstage-cli package build",
     "clean": "backstage-cli package clean",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz",
     "generate": "backstage-repo-tools package schema openapi generate --server",
     "lint": "backstage-cli package lint",
     "prepack": "backstage-cli package prepack",

yarn.lock

@@ -9116,6 +9116,7 @@ __metadata:
     "@backstage/plugin-permission-node": "workspace:^"
     "@backstage/plugin-search-backend-node": "workspace:^"
     "@backstage/plugin-search-common": "workspace:^"
+    "@backstage/repo-tools": "workspace:^"
     "@backstage/types": "workspace:^"
     "@types/express": ^4.17.6
     "@types/supertest": ^2.0.8
@@ -10112,6 +10113,7 @@ __metadata:
     "@backstage/cli": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/cli-node": "workspace:^"
+    "@backstage/config-loader": "workspace:^"
     "@backstage/errors": "workspace:^"
     "@backstage/types": "workspace:^"
     "@manypkg/get-packages": ^1.1.3
@@ -10130,6 +10132,7 @@ __metadata:
     "@types/prettier": ^2.0.0
     chalk: ^4.0.0
     codeowners-utils: ^1.0.2
+    command-exists: ^1.2.9
     commander: ^12.0.0
     fs-extra: ^11.2.0
     glob: ^8.0.3