Merge pull request #23084 from aramissennyeydd/openapi-tooling/schemathesis

feat(openapi-tooling): add support for fuzzing with schemathesis

.changeset/fuzzy-beans-occur.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Fixes an issue where `/analyze-location` would incorrectly throw a 500 error on an invalid url.

.changeset/violet-lobsters-unite.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': minor
+---
+
+Adds two new commands, `repo schema openapi fuzz` and `package schema openapi fuzz` for fuzzing your plugins documented with OpenAPI. This can help find bugs in your application code through the use of auto-generated schema-compliant inputs. For more information on the underlying library this leverages, take a look at [the docs](https://schemathesis.readthedocs.io/en/stable/index.html).

.gitignore

@@ -166,3 +166,7 @@ plugins-report.csv
 
 # Temporary knip configs
 knip.json
+
+# Schemathesis temporary files
+.hypothesis/
+.cassettes/

packages/repo-tools/cli-report.md

@@ -97,9 +97,23 @@ Options:
 Commands:
   init
   generate [options]
+  fuzz [options]
   help [command]
 ```
 
+### `backstage-repo-tools package schema openapi fuzz`
+
+```
+Usage: backstage-repo-tools package schema openapi fuzz [options]
+
+Options:
+  --limit <limit>
+  --workers <workers>
+  --debug
+  --exclude-checks <excludeChecks>
+  -h, --help
+```
+
 ### `backstage-repo-tools package schema openapi generate`
 
 ```
@@ -157,9 +171,20 @@ Commands:
   verify [paths...]
   lint [options] [paths...]
   test [options] [paths...]
+  fuzz [options]
   help [command]
 ```
 
+### `backstage-repo-tools repo schema openapi fuzz`
+
+```
+Usage: backstage-repo-tools repo schema openapi fuzz [options]
+
+Options:
+  --since <ref>
+  -h, --help
+```
+
 ### `backstage-repo-tools repo schema openapi lint`
 
 ```

packages/repo-tools/package.json

@@ -36,6 +36,7 @@
     "@backstage/catalog-model": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/cli-node": "workspace:^",
+    "@backstage/config-loader": "workspace:^",
     "@backstage/errors": "workspace:^",
     "@manypkg/get-packages": "^1.1.3",
     "@microsoft/api-documenter": "^7.22.33",
@@ -50,6 +51,7 @@
     "@stoplight/types": "^14.0.0",
     "chalk": "^4.0.0",
     "codeowners-utils": "^1.0.2",
+    "command-exists": "^1.2.9",
     "commander": "^12.0.0",
     "fs-extra": "^11.2.0",
     "glob": "^8.0.3",

packages/repo-tools/src/commands/index.ts

@@ -59,6 +59,25 @@ function registerPackageCommand(program: Command) {
         import('./package/schema/openapi/generate').then(m => m.command),
       ),
     );
+
+  openApiCommand
+    .command('fuzz')
+    .description(
+      'Fuzz an OpenAPI schema by generating random data and sending it to the server.',
+    )
+    .option('--limit <limit>', 'Maximum number of requests to send.')
+    .option('--workers <workers>', 'Number of workers to use', '2')
+    .option(
+      '--debug',
+      `Enable debug mode, which will save cassettes to '.cassettes/{pluginId}.yml'`,
+    )
+    .option(
+      '--exclude-checks <excludeChecks>',
+      'Exclude checks from schemathesis run',
+    )
+    .action(
+      lazy(() => import('./package/schema/openapi/fuzz').then(m => m.command)),
+    );
 }
 
 function registerRepoCommand(program: Command) {
@@ -103,6 +122,17 @@ function registerRepoCommand(program: Command) {
     .action(
       lazy(() => import('./repo/schema/openapi/test').then(m => m.bulkCommand)),
     );
+
+  openApiCommand
+    .command('fuzz')
+    .description('Fuzz all packages')
+    .option(
+      '--since <ref>',
+      'Only fuzz packages that have changed since the given ref',
+    )
+    .action(
+      lazy(() => import('./repo/schema/openapi/fuzz').then(m => m.command)),
+    );
 }
 
 export function registerCommands(program: Command) {

packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import { paths as cliPaths } from '../../../../lib/paths';
+import chalk from 'chalk';
+import { spawn } from '../../../../lib/exec';
+import { getPathToCurrentOpenApiSpec } from '../../../../lib/openapi/helpers';
+import { ConfigSources } from '@backstage/config-loader';
+import YAML from 'js-yaml';
+import { join } from 'path';
+import { OptionValues } from 'commander';
+import { sync as existsSync } from 'command-exists';
+
+async function fuzz(opts: OptionValues) {
+  const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+
+  if (!existsSync('st')) {
+    console.log(
+      chalk.red(
+        `Please install schemathesis globally with 'python -m pip install schemathesis'. Then run this command again.`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  const openapiSpec = YAML.load(
+    await fs.readFile(resolvedOpenapiPath, 'utf8'),
+  ) as { info: { title: string } };
+  const configSource = ConfigSources.default({
+    rootDir: cliPaths.targetRoot,
+  });
+  const config = await ConfigSources.toConfig(configSource);
+  const pluginId = openapiSpec.info.title;
+  const args = [];
+  if (opts.debug) {
+    args.push(
+      '--cassette-path',
+      cliPaths.resolveTargetRoot(join('.cassettes', `${pluginId}.yml`)),
+    );
+  }
+
+  if (opts.limit) {
+    args.push('--hypothesis-max-examples', opts.limit);
+  }
+  args.push('--workers', opts.workers);
+
+  if (opts.useGuest) {
+    // TODO: @sennyeya This should leverage the `guest-provider` if available.
+    args.push('--header', `Authorization: Basic guest`);
+  } else {
+    // This is just here to prevent any "Invalid JWT" errors during execution.
+    args.push('--header', `Authorization: Basic test`);
+  }
+
+  if (opts.excludeChecks) {
+    args.push('--exclude-checks', opts.excludeChecks);
+  }
+
+  await spawn(
+    'st',
+    [
+      'run',
+      '--checks',
+      'all',
+      ...args,
+      `${config.getString('backend.baseUrl')}/api/${pluginId}/openapi.json`,
+    ],
+    {
+      stdio: ['ignore', 'inherit', 'ignore'],
+    },
+  );
+}
+
+export async function command(opts: OptionValues) {
+  try {
+    await fuzz(opts);
+    console.log(chalk.green(`Successfully fuzzed.`));
+  } catch (err) {
+    console.log(chalk.red(`OpenAPI fuzzing failed.`));
+    console.error(err);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PackageGraph } from '@backstage/cli-node';
+import { OptionValues } from 'commander';
+import { exec } from '../../../../lib/exec';
+import chalk from 'chalk';
+
+export async function command(opts: OptionValues) {
+  let packages = await PackageGraph.listTargetPackages();
+  if (opts.since) {
+    const graph = PackageGraph.fromPackages(packages);
+    const changedPackages = await graph.listChangedPackages({
+      ref: opts.since,
+      analyzeLockfile: true,
+    });
+    const withDevDependents = graph.collectPackageNames(
+      changedPackages.map(pkg => pkg.name),
+      pkg => pkg.localDevDependents.keys(),
+    );
+    packages = Array.from(withDevDependents).map(name => graph.get(name)!);
+  }
+
+  const fuzzablePackages = packages.filter(e => e.packageJson.scripts?.fuzz);
+  try {
+    for (const pkg of fuzzablePackages) {
+      await exec('yarn', ['fuzz'], {
+        cwd: pkg.dir,
+      });
+    }
+    console.log(chalk.green(`Successfully fuzzed.`));
+  } catch (err) {
+    console.error(err.stdout);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/lib/exec.ts

@@ -14,20 +14,50 @@
  * limitations under the License.
  */
 import { promisify } from 'util';
-import { ExecOptions, exec as execCb } from 'child_process';
+import {
+  ExecOptions,
+  SpawnOptions,
+  exec as execCb,
+  spawn as spawnOriginal,
+} from 'child_process';
 
 const execPromise = promisify(execCb);
 
 export const exec = (
   command: string,
-  options: string[] = [],
-  execOptions?: ExecOptions,
+  args: string[] = [],
+  options?: ExecOptions,
 ) => {
   return execPromise(
     [
       command,
-      ...options.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)),
+      ...args.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)),
     ].join(' '),
-    execOptions,
+    options,
   );
 };
+
+export const spawn = (
+  command: string,
+  args: string[],
+  options?: SpawnOptions,
+) => {
+  return new Promise((resolve, reject) => {
+    const cp = spawnOriginal(command, args, options ?? {});
+    const error: string[] = [];
+    const stdout: string[] = [];
+
+    cp.stdout?.on('data', data => {
+      stdout.push(data.toString());
+    });
+
+    cp.on('error', e => {
+      error.push(e.toString());
+    });
+
+    cp.on('close', exitCode => {
+      if (exitCode) reject(error.join(''));
+      else resolve(stdout.join(''));
+    });
+  });
+};

plugins/catalog-backend/package.json

@@ -43,6 +43,7 @@
   "scripts": {
     "build": "backstage-cli package build",
     "clean": "backstage-cli package clean",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz --exclude-checks response_schema_conformance",
     "generate": "backstage-repo-tools package schema openapi generate --server --client-package packages/catalog-client",
     "lint": "backstage-cli package lint",
     "prepack": "backstage-cli package prepack",

plugins/catalog-backend/src/schema/openapi.generated.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 The Backstage Authors
+ * Copyright 2024 The Backstage Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -1540,7 +1540,7 @@ export const spec = {
           '400': {
             description: 'Validation errors.',
             content: {
-              'application/json; charset=utf-8': {
+              'application/json': {
                 schema: {
                   type: 'object',
                   properties: {

plugins/catalog-backend/src/schema/openapi.yaml

@@ -1135,7 +1135,7 @@ paths:
         '400':
           description: Validation errors.
           content:
-            application/json; charset=utf-8:
+            application/json:
               schema:
                 type: object
                 properties:

plugins/catalog-backend/src/service/createRouter.test.ts

@@ -42,6 +42,7 @@ import { decodeCursor, encodeCursor } from './util';
 import { wrapInOpenApiTestServer } from '@backstage/backend-openapi-utils';
 import { Server } from 'http';
 import { mockCredentials, mockServices } from '@backstage/backend-test-utils';
+import { LocationAnalyzer } from '../ingestion';
 
 describe('createRouter readonly disabled', () => {
   let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
@@ -49,6 +50,7 @@ describe('createRouter readonly disabled', () => {
   let orchestrator: jest.Mocked<CatalogProcessingOrchestrator>;
   let app: express.Express | Server;
   let refreshService: RefreshService;
+  let locationAnalyzer: jest.Mocked<LocationAnalyzer>;
 
   beforeAll(async () => {
     entitiesCatalog = {
@@ -66,6 +68,10 @@ describe('createRouter readonly disabled', () => {
       deleteLocation: jest.fn(),
       getLocationByEntity: jest.fn(),
     };
+
+    locationAnalyzer = {
+      analyzeLocation: jest.fn(),
+    };
     refreshService = { refresh: jest.fn() };
     orchestrator = { process: jest.fn() };
     const router = await createRouter({
@@ -78,6 +84,7 @@ describe('createRouter readonly disabled', () => {
       permissionIntegrationRouter: express.Router(),
       auth: mockServices.auth(),
       httpAuth: mockServices.httpAuth(),
+      locationAnalyzer,
     });
     app = wrapInOpenApiTestServer(express().use(router));
   });
@@ -821,6 +828,21 @@ describe('createRouter readonly disabled', () => {
       });
     });
   });
+
+  describe('POST /analyze-location', () => {
+    it('handles invalid URLs', async () => {
+      const parseUrlError = new Error();
+      (parseUrlError as any).subject_url = 'not a url';
+      locationAnalyzer.analyzeLocation.mockRejectedValue(parseUrlError);
+      const response = await request(app)
+        .post('/analyze-location')
+        .send({ location: { type: 'url', target: 'not a url' } });
+      expect(response.status).toEqual(400);
+      expect(response.body.error.message).toMatch(
+        /The given location.target is not a URL/,
+      );
+    });
+  });
 });
 
 describe('createRouter readonly enabled', () => {

plugins/catalog-backend/src/service/createRouter.ts

@@ -23,7 +23,7 @@ import {
   stringifyEntityRef,
 } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
-import { NotFoundError, serializeError } from '@backstage/errors';
+import { InputError, NotFoundError, serializeError } from '@backstage/errors';
 import express from 'express';
 import { Logger } from 'winston';
 import yn from 'yn';
@@ -298,8 +298,20 @@ export async function createRouter(
         location: locationInput,
         catalogFilename: z.string().optional(),
       });
-      const output = await locationAnalyzer.analyzeLocation(schema.parse(body));
-      res.status(200).json(output);
+      const parsedBody = schema.parse(body);
+      try {
+        const output = await locationAnalyzer.analyzeLocation(parsedBody);
+        res.status(200).json(output);
+      } catch (err) {
+        if (
+          // Catch errors from parse-url library.
+          err.name === 'Error' &&
+          'subject_url' in err
+        ) {
+          throw new InputError('The given location.target is not a URL');
+        }
+        throw err;
+      }
     });
   }
 

plugins/search-backend/package.json

@@ -40,7 +40,8 @@
     "prepack": "backstage-cli package prepack",
     "postpack": "backstage-cli package postpack",
     "clean": "backstage-cli package clean",
-    "generate": "backstage-repo-tools package schema openapi generate --server"
+    "generate": "backstage-repo-tools package schema openapi generate --server",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz"
   },
   "dependencies": {
     "@backstage/backend-common": "workspace:^",
@@ -52,6 +53,7 @@
     "@backstage/plugin-permission-node": "workspace:^",
     "@backstage/plugin-search-backend-node": "workspace:^",
     "@backstage/plugin-search-common": "workspace:^",
+    "@backstage/repo-tools": "workspace:^",
     "@backstage/types": "workspace:^",
     "@types/express": "^4.17.6",
     "dataloader": "^2.0.0",

plugins/todo-backend/package.json

@@ -25,6 +25,7 @@
   "scripts": {
     "build": "backstage-cli package build",
     "clean": "backstage-cli package clean",
+    "fuzz": "backstage-repo-tools package schema openapi fuzz",
     "generate": "backstage-repo-tools package schema openapi generate --server",
     "lint": "backstage-cli package lint",
     "prepack": "backstage-cli package prepack",

yarn.lock

@@ -8939,6 +8939,7 @@ __metadata:
     "@backstage/plugin-permission-node": "workspace:^"
     "@backstage/plugin-search-backend-node": "workspace:^"
     "@backstage/plugin-search-common": "workspace:^"
+    "@backstage/repo-tools": "workspace:^"
     "@backstage/types": "workspace:^"
     "@types/express": ^4.17.6
     "@types/supertest": ^2.0.8
@@ -9926,6 +9927,7 @@ __metadata:
     "@backstage/cli": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/cli-node": "workspace:^"
+    "@backstage/config-loader": "workspace:^"
     "@backstage/errors": "workspace:^"
     "@backstage/types": "workspace:^"
     "@manypkg/get-packages": ^1.1.3
@@ -9944,6 +9946,7 @@ __metadata:
     "@types/prettier": ^2.0.0
     chalk: ^4.0.0
     codeowners-utils: ^1.0.2
+    command-exists: ^1.2.9
     commander: ^12.0.0
     fs-extra: ^11.2.0
     glob: ^8.0.3