From 317cb08b5d54677d513e375313b25cec32837a34 Mon Sep 17 00:00:00 2001 From: aramissennyeydd Date: Mon, 19 Feb 2024 22:07:46 -0500 Subject: [PATCH] feat(openapi-tooling): add support for fuzzing with schemathesis Signed-off-by: aramissennyeydd --- .gitignore | 4 + packages/repo-tools/package.json | 2 + packages/repo-tools/src/commands/index.ts | 27 ++++++ .../commands/package/schema/openapi/fuzz.ts | 96 +++++++++++++++++++ .../src/commands/repo/schema/openapi/fuzz.ts | 48 ++++++++++ packages/repo-tools/src/lib/exec.ts | 40 +++++++- plugins/catalog-backend/package.json | 1 + .../src/service/createRouter.ts | 3 +- plugins/catalog-backend/src/service/util.ts | 2 +- plugins/search-backend/package.json | 4 +- plugins/todo-backend/package.json | 1 + yarn.lock | 3 + 12 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts create mode 100644 packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts diff --git a/.gitignore b/.gitignore index e499faf29e..5ba3da3148 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,7 @@ plugins-report.csv # Temporary knip configs knip.json + +# Schemathesis temporary files +.hypothesis/ +.cassettes/ diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 73195e8816..7f085da5eb 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -36,6 +36,7 @@ "@backstage/catalog-model": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/cli-node": "workspace:^", + "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", "@manypkg/get-packages": "^1.1.3", "@microsoft/api-documenter": "^7.22.33", @@ -50,6 +51,7 @@ "@stoplight/types": "^14.0.0", "chalk": "^4.0.0", "codeowners-utils": "^1.0.2", + "command-exists": "^1.2.9", "commander": "^12.0.0", "fs-extra": "^11.2.0", "glob": "^8.0.3", diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index e0463da4f9..2c3d96a693 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -59,6 +59,22 @@ function registerPackageCommand(program: Command) { import('./package/schema/openapi/generate').then(m => m.command), ), ); + + openApiCommand + .command('fuzz') + .description( + 'Fuzz an OpenAPI schema by generating random data and sending it to the server.', + ) + .option('--count ', 'Number of requests to send') + .option('--workers ', 'Number of workers to use', '2') + .option('--debug', 'Enable debug mode') + .option( + '--exclude-checks ', + 'Exclude checks from schemathesis run', + ) + .action( + lazy(() => import('./package/schema/openapi/fuzz').then(m => m.command)), + ); } function registerRepoCommand(program: Command) { @@ -103,6 +119,17 @@ function registerRepoCommand(program: Command) { .action( lazy(() => import('./repo/schema/openapi/test').then(m => m.bulkCommand)), ); + + openApiCommand + .command('fuzz') + .description('Fuzz all packages') + .option( + '--since ', + 'Only fuzz packages that have changed since the given ref', + ) + .action( + lazy(() => import('./repo/schema/openapi/fuzz').then(m => m.command)), + ); } export function registerCommands(program: Command) { diff --git a/packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts b/packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts new file mode 100644 index 0000000000..7bdf9ccfd2 --- /dev/null +++ b/packages/repo-tools/src/commands/package/schema/openapi/fuzz.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs-extra'; +import { paths as cliPaths } from '../../../../lib/paths'; +import chalk from 'chalk'; +import { spawn } from '../../../../lib/exec'; +import { getPathToCurrentOpenApiSpec } from '../../../../lib/openapi/helpers'; +import { ConfigSources } from '@backstage/config-loader'; +import YAML from 'js-yaml'; +import { join } from 'path'; +import { OptionValues } from 'commander'; +import { sync as existsSync } from 'command-exists'; + +async function fuzz(opts: OptionValues) { + const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec(); + + if (!existsSync('st')) { + console.log( + chalk.red( + `Please install schemathesis globally with 'python -m pip install schemathesis'. Then run this command again.`, + ), + ); + process.exit(1); + } + + const openapiSpec = YAML.load( + await fs.readFile(resolvedOpenapiPath, 'utf8'), + ) as { info: { title: string } }; + const configSource = ConfigSources.default({ + rootDir: cliPaths.targetRoot, + }); + const config = await ConfigSources.toConfig(configSource); + const pluginId = openapiSpec.info.title; + const args = []; + if (opts.debug) { + args.push( + '--cassette-path', + cliPaths.resolveTargetRoot(join('.cassettes', `${pluginId}`)), + ); + } + + if (opts.count) { + args.push('--hypothesis-max-examples', opts.count); + } + args.push('--workers', opts.workers); + + if (opts.useGuest) { + args.push('--header', `Authorization: Basic guest`); + } else { + args.push('--header', `Authorization: Basic test`); + } + + if (opts.excludeChecks) { + args.push('--exclude-checks', opts.excludeChecks); + } + + await spawn( + 'st', + [ + 'run', + '--checks', + 'all', + '--data-generation-method', + 'all', + ...args, + `${config.getString('backend.baseUrl')}/api/${pluginId}/openapi.json`, + ], + { + stdio: ['ignore', 'inherit', 'inherit'], + }, + ); +} + +export async function command(opts: OptionValues) { + try { + await fuzz(opts); + console.log(chalk.green(`Successfully fuzzed.`)); + } catch (err) { + console.log(chalk.red(`OpenAPI fuzzing failed.`)); + console.error(err); + process.exit(1); + } +} diff --git a/packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts b/packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts new file mode 100644 index 0000000000..7767eb4a8a --- /dev/null +++ b/packages/repo-tools/src/commands/repo/schema/openapi/fuzz.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PackageGraph } from '@backstage/cli-node'; +import { OptionValues } from 'commander'; +import { exec } from '../../../../lib/exec'; +import chalk from 'chalk'; + +export async function command(opts: OptionValues) { + let packages = await PackageGraph.listTargetPackages(); + if (opts.since) { + const graph = PackageGraph.fromPackages(packages); + const changedPackages = await graph.listChangedPackages({ + ref: opts.since, + analyzeLockfile: true, + }); + const withDevDependents = graph.collectPackageNames( + changedPackages.map(pkg => pkg.name), + pkg => pkg.localDevDependents.keys(), + ); + packages = Array.from(withDevDependents).map(name => graph.get(name)!); + } + + const fuzzablePackages = packages.filter(e => e.packageJson.scripts?.fuzz); + try { + for (const pkg of fuzzablePackages) { + await exec('yarn', ['fuzz'], { + cwd: pkg.dir, + }); + } + console.log(chalk.green(`Successfully fuzzed.`)); + } catch (err) { + console.error(err.stdout); + process.exit(1); + } +} diff --git a/packages/repo-tools/src/lib/exec.ts b/packages/repo-tools/src/lib/exec.ts index 7f1bfe2302..248ed34f22 100644 --- a/packages/repo-tools/src/lib/exec.ts +++ b/packages/repo-tools/src/lib/exec.ts @@ -14,20 +14,50 @@ * limitations under the License. */ import { promisify } from 'util'; -import { ExecOptions, exec as execCb } from 'child_process'; +import { + ExecOptions, + SpawnOptions, + exec as execCb, + spawn as spawnOriginal, +} from 'child_process'; const execPromise = promisify(execCb); export const exec = ( command: string, - options: string[] = [], - execOptions?: ExecOptions, + args: string[] = [], + options?: ExecOptions, ) => { return execPromise( [ command, - ...options.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)), + ...args.filter(e => e).map(e => (e.startsWith('-') ? e : `"${e}"`)), ].join(' '), - execOptions, + options, ); }; + +export const spawn = ( + command: string, + args: string[], + options?: SpawnOptions, +) => { + return new Promise((resolve, reject) => { + const cp = spawnOriginal(command, args, options ?? {}); + const error: string[] = []; + const stdout: string[] = []; + + cp.stdout?.on('data', data => { + stdout.push(data.toString()); + }); + + cp.on('error', e => { + error.push(e.toString()); + }); + + cp.on('close', exitCode => { + if (exitCode) reject(error.join('')); + else resolve(stdout.join('')); + }); + }); +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 54523ad909..e387fbfc77 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -43,6 +43,7 @@ "scripts": { "build": "backstage-cli package build", "clean": "backstage-cli package clean", + "fuzz": "backstage-repo-tools package schema openapi fuzz --exclude-checks content_type_conformance,response_schema_conformance", "generate": "backstage-repo-tools package schema openapi generate --server --client-package packages/catalog-client", "lint": "backstage-cli package lint", "prepack": "backstage-cli package prepack", diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 740ef63ea9..834e0953bb 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -298,7 +298,8 @@ export async function createRouter( location: locationInput, catalogFilename: z.string().optional(), }); - const output = await locationAnalyzer.analyzeLocation(schema.parse(body)); + const parsedBody = schema.parse(body); + const output = await locationAnalyzer.analyzeLocation(parsedBody); res.status(200).json(output); }); } diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 4b041d1a70..6ad53c9050 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -50,7 +50,7 @@ export async function requireRequestBody(req: Request): Promise { export const locationInput = z .object({ type: z.string(), - target: z.string(), + target: z.string().url(), presence: z.literal('required').or(z.literal('optional')).optional(), }) .strict(); // no unknown keys; diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index 8c255cc919..61eef43980 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -40,7 +40,8 @@ "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", "clean": "backstage-cli package clean", - "generate": "backstage-repo-tools package schema openapi generate --server" + "generate": "backstage-repo-tools package schema openapi generate --server", + "fuzz": "backstage-repo-tools package schema openapi fuzz" }, "dependencies": { "@backstage/backend-common": "workspace:^", @@ -52,6 +53,7 @@ "@backstage/plugin-permission-node": "workspace:^", "@backstage/plugin-search-backend-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", + "@backstage/repo-tools": "workspace:^", "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "dataloader": "^2.0.0", diff --git a/plugins/todo-backend/package.json b/plugins/todo-backend/package.json index 1716295c2a..a201c6a8d7 100644 --- a/plugins/todo-backend/package.json +++ b/plugins/todo-backend/package.json @@ -25,6 +25,7 @@ "scripts": { "build": "backstage-cli package build", "clean": "backstage-cli package clean", + "fuzz": "backstage-repo-tools package schema openapi fuzz", "generate": "backstage-repo-tools package schema openapi generate --server", "lint": "backstage-cli package lint", "prepack": "backstage-cli package prepack", diff --git a/yarn.lock b/yarn.lock index 11205a2254..6778003ffa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9116,6 +9116,7 @@ __metadata: "@backstage/plugin-permission-node": "workspace:^" "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" + "@backstage/repo-tools": "workspace:^" "@backstage/types": "workspace:^" "@types/express": ^4.17.6 "@types/supertest": ^2.0.8 @@ -10112,6 +10113,7 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/cli-node": "workspace:^" + "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" "@manypkg/get-packages": ^1.1.3 @@ -10130,6 +10132,7 @@ __metadata: "@types/prettier": ^2.0.0 chalk: ^4.0.0 codeowners-utils: ^1.0.2 + command-exists: ^1.2.9 commander: ^12.0.0 fs-extra: ^11.2.0 glob: ^8.0.3