Merge pull request #23370 from aramissennyeydd/openapi-tooling/breaking-changes

feat(openapi-tooling): add checks for API breaking changes

.changeset/flat-countries-clap.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': minor
+---
+
+Adds 2 new commands `repo schema openapi diff` and `package schema openapi diff`. `repo schema openapi diff` is intended to power a new breaking changes check on pull requests and the package level command allows plugin developers to quickly see new API breaking changes. They're intended to be used in complement with the existing `repo schema openapi verify` command to validate your OpenAPI spec against a variety of things.

.github/workflows/api-breaking-changes-comment.yml

@@ -0,0 +1,111 @@
+name: API Breaking Changes (comment)
+
+on:
+  workflow_run:
+    workflows:
+      - 'API Breaking Changes (Trigger)'
+    types:
+      - completed
+
+jobs:
+  setup:
+    name: Add values from previous step
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    permissions:
+      # "If you specify the access for any of these scopes, all of those that are not specified are set to none."
+      # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+      actions: read # Access cache
+    outputs:
+      git-ref: ${{ steps.event.outputs.GIT_REF }}
+      pr-number: ${{ steps.event.outputs.PR_NUMBER }}
+      action: ${{ steps.event.outputs.ACTION }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          disable-sudo: true
+          egress-policy: block
+          allowed-endpoints: >
+            api.github.com:443
+
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            if (matchArtifact === undefined) {
+              throw TypeError('Build Artifact not found!');
+            }
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+
+      - name: 'Accept event from first stage'
+        run: unzip preview-spec.zip event.json
+
+      - name: Read Event into ENV
+        id: event
+        run: |
+          echo PR_NUMBER=$(jq '.number | tonumber' < event.json) >> $GITHUB_OUTPUT
+          echo ACTION=$(jq --raw-output '.action | tostring | [scan("\\w+")][0]' < event.json) >> $GITHUB_OUTPUT
+          echo GIT_REF=$(jq --raw-output '.pull_request.head.sha | tostring | [scan("\\w+")][0]' < event.json) >> $GITHUB_OUTPUT
+
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ steps.event.outputs.PR_NUMBER }}"
+          echo "Git Ref: ${{ steps.event.outputs.GIT_REF }}"
+          echo "Action: ${{ steps.event.outputs.ACTION }}"
+          cat event.json
+
+      - name: Get Comment
+        id: get-comment
+        run: |
+          unzip preview-spec.zip comment.md
+          ls
+          grep
+
+  add-comment:
+    name: Write comment about issues
+    needs:
+      - setup
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      # Identify comment to be updated
+      - name: Find comment for API Changes
+        uses: peter-evans/find-comment@d5fe37641ad8451bdd80312415672ba26c86575e # v3
+        id: find-comment
+        with:
+          issue-number: ${{ needs.setup.outputs.pr-number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: API changes
+          direction: last
+
+      - name: Create or Update Comment with API Changes
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: comment.md
+          edit-mode: replace

.github/workflows/api-breaking-changes.yml

@@ -0,0 +1,56 @@
+name: API Breaking Changes (Trigger)
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
+    paths:
+      - '**/openapi.yaml'
+
+jobs:
+  get-backstage-changes:
+    env:
+      NODE_OPTIONS: --max-old-space-size=4096
+    name: Build PR image
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          # Fetch the commit that's merged into the base rather than the target ref
+          # This will let us diff only the contents of the PR, without fetching more history
+          ref: 'refs/pull/${{ github.event.pull_request.number }}/merge'
+      - name: fetch base
+        run: git fetch --depth 1 origin ${{ github.base_ref }}
+
+      - name: setup-node
+        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        with:
+          node-version: 18.x
+          registry-url: https://registry.npmjs.org/
+
+      - name: yarn install
+        uses: backstage/actions/yarn-install@a674369920067381b450d398b27df7039b7ef635 # v0.6.5
+        with:
+          cache-prefix: linux-v18
+
+      - name: breaking changes check
+        run: |
+          yarn backstage-repo-tools repo schema openapi check --since origin/${{ github.base_ref }} > comment.md
+
+      - name: Upload Rendered Comment as Artifact
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3
+        with:
+          name: preview-spec
+          path: comment.md
+          retention-days: 2
+
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        with:
+          name: preview-spec
+          path: ${{ github.event_path }}
+          retention-days: 2

NOTICE

@@ -5,3 +5,4 @@ Portions of this software were developed by third-party software vendors:
 
 - Tech Radar Plugin (https://opensource.zalando.com/tech-radar/), Copyright (c) 2017 Zalando SE
 - [OpenAPI Generator Templates](./packages/repo-tools/templates), Copyright 2018 OpenAPI-Generator Contributors (https://openapi-generator.tech) Copyright 2018 SmartBear Software
+- Optic CLI (https://github.com/opticdev/optic), Copyright 2022, Optic Labs Corporation

packages/repo-tools/cli-report.md

@@ -172,9 +172,20 @@ Commands:
   lint [options] [paths...]
   test [options] [paths...]
   fuzz [options]
+  diff [options]
   help [command]
 ```
 
+### `backstage-repo-tools repo schema openapi diff`
+
+```
+Usage: backstage-repo-tools repo schema openapi diff [options]
+
+Options:
+  --since <ref>
+  -h, --help
+```
+
 ### `backstage-repo-tools repo schema openapi fuzz`
 
 ```

packages/repo-tools/package.json

@@ -49,6 +49,7 @@
     "@stoplight/spectral-rulesets": "^1.18.0",
     "@stoplight/spectral-runtime": "^1.1.2",
     "@stoplight/types": "^14.0.0",
+    "@useoptic/openapi-utilities": "^0.54.8",
     "chalk": "^4.0.0",
     "codeowners-utils": "^1.0.2",
     "command-exists": "^1.2.9",

packages/repo-tools/src/commands/index.ts

@@ -78,6 +78,15 @@ function registerPackageCommand(program: Command) {
     .action(
       lazy(() => import('./package/schema/openapi/fuzz').then(m => m.command)),
     );
+
+  openApiCommand
+    .command('diff')
+    .option('--ignore', 'Ignore linting failures and only log the results.')
+    .option('--json', 'Output the results as JSON')
+    .option('--since <ref>', 'Diff the API against a specific ref')
+    .action(
+      lazy(() => import('./package/schema/openapi/diff').then(m => m.command)),
+    );
 }
 
 function registerRepoCommand(program: Command) {
@@ -96,7 +105,7 @@ function registerRepoCommand(program: Command) {
   openApiCommand
     .command('verify [paths...]')
     .description(
-      'Verify that all OpenAPI schemas are valid and have a matching `schemas/openapi.generated.ts` file.',
+      'Verify that all OpenAPI schemas are valid and set up correctly.',
     )
     .action(
       lazy(() =>
@@ -133,6 +142,20 @@ function registerRepoCommand(program: Command) {
     .action(
       lazy(() => import('./repo/schema/openapi/fuzz').then(m => m.command)),
     );
+
+  openApiCommand
+    .command('diff')
+    .description(
+      'Diff the repository against a specific ref, will run all package `diff` scripts.',
+    )
+    .option(
+      '--since <ref>',
+      'Diff the API against a specific ref',
+      'origin/master',
+    )
+    .action(
+      lazy(() => import('./repo/schema/openapi/diff').then(m => m.command)),
+    );
 }
 
 export function registerCommands(program: Command) {

packages/repo-tools/src/commands/package/schema/openapi/diff.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import chalk from 'chalk';
+import { exec } from '../../../../lib/exec';
+import { getPathToCurrentOpenApiSpec } from '../../../../lib/openapi/helpers';
+import { paths as cliPaths } from '../../../../lib/paths';
+import { OptionValues } from 'commander';
+import { env } from 'process';
+import { readFile, rm } from 'fs/promises';
+import { resolve } from 'path';
+
+const reduceOpticOutput = (output: string) => {
+  return output
+    .split('\n')
+    .filter(e => !e.startsWith('Rerun') && e.trim())
+    .join('\n');
+};
+
+async function check(opts: OptionValues) {
+  const resolvedOpenapiPath = await getPathToCurrentOpenApiSpec();
+
+  let baseRef = opts.since;
+  if (!baseRef) {
+    const { stdout: branch } = await exec(
+      'git merge-base --fork-point origin/master',
+    );
+    baseRef = branch.toString().trim();
+  }
+
+  let failed = false;
+  let output = '';
+  try {
+    const { stdout } = await exec(
+      'yarn optic diff',
+      [
+        resolvedOpenapiPath,
+        '--check',
+        opts.json ? '--json' : '',
+        '--base',
+        baseRef,
+      ],
+      {
+        cwd: cliPaths.targetRoot,
+        env: { CI: opts.json ? '1' : undefined, ...env },
+      },
+    );
+    output = stdout.toString();
+  } catch (err) {
+    output = err.stdout;
+    failed = true;
+  }
+
+  if (opts.json) {
+    const file = (
+      await readFile(resolve(cliPaths.targetRoot, 'ci-run-details.json'))
+    ).toString();
+    const results = JSON.parse(file);
+    console.log(file);
+    if (!opts.ignore && results.failed) {
+      throw new Error('Some checks failed');
+    }
+
+    await rm(resolve(cliPaths.targetRoot, 'ci-run-details.json'));
+  } else {
+    console.log(reduceOpticOutput(output));
+    if (!opts.ignore && failed) {
+      throw new Error('Some checks failed');
+    }
+  }
+}
+
+export async function command(opts: OptionValues) {
+  try {
+    await check(opts);
+    if (!opts.json) console.log(chalk.green(`All checks passed.`));
+  } catch (err) {
+    if (!opts.json) console.log(chalk.red(err.message));
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/diff.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PackageGraph } from '@backstage/cli-node';
+import { OptionValues } from 'commander';
+import { exec } from '../../../../lib/exec';
+import {
+  CiRunDetails,
+  generateCompareSummaryMarkdown,
+} from '../../../../lib/openapi/optic/helpers';
+import { paths as cliPaths } from '../../../../lib/paths';
+import { YAML_SCHEMA_PATH } from '../../../../lib/openapi/constants';
+
+function cleanUpApiName(e: { apiName: string }) {
+  e.apiName = e.apiName
+    .replace(cliPaths.targetDir, '')
+    .replace(YAML_SCHEMA_PATH, '');
+}
+
+export async function command(opts: OptionValues) {
+  let packages = await PackageGraph.listTargetPackages();
+
+  let since = '';
+  if (opts.since) {
+    const { stdout: sinceRaw } = await exec('git', ['rev-parse', opts.since]);
+    since = sinceRaw.toString().trim();
+    const { stdout: changedFilesRaw } = await exec('git', [
+      'diff',
+      '--name-only',
+      since,
+    ]);
+    const changedFiles = changedFilesRaw.toString().trim();
+
+    const changedOpenApiSpecs = changedFiles
+      .split('\n')
+      .filter(e => e.endsWith(YAML_SCHEMA_PATH))
+      .map(e => cliPaths.resolveTarget(e));
+
+    // filter packages by changedFiles
+    packages = packages.filter(pkg =>
+      changedOpenApiSpecs.some(e => e.startsWith(`${pkg.dir}/`)),
+    );
+  }
+
+  const checkablePackages = packages.filter(e => e.packageJson.scripts?.diff);
+
+  try {
+    const outputs = {
+      completed: [],
+      failed: [],
+      noop: [],
+      warning: [],
+      severity: 0,
+    } as CiRunDetails;
+    for (const pkg of checkablePackages) {
+      const sinceCommands = since ? ['--since', since] : [];
+      const { stdout } = await exec(
+        'yarn',
+        ['diff', '--ignore', '--json', ...sinceCommands],
+        {
+          cwd: pkg.dir,
+        },
+      );
+      const result = JSON.parse(stdout.toString());
+      outputs.completed.push(...(result.completed ?? []));
+      outputs.failed.push(...(result.failed ?? []));
+      outputs.noop.push(...(result.noop ?? []));
+    }
+
+    for (const pkg of packages.filter(e => !e.packageJson.scripts?.diff)) {
+      outputs.warning?.push({
+        apiName: `${pkg.dir}/`,
+        warning: 'No diff script found in package.json',
+      });
+    }
+
+    outputs.completed.forEach(cleanUpApiName);
+    outputs.failed.forEach(cleanUpApiName);
+    outputs.noop.forEach(cleanUpApiName);
+    outputs.warning?.forEach(cleanUpApiName);
+
+    const { stdout: currentSha } = await exec('git', ['rev-parse', 'HEAD']);
+    console.log(
+      generateCompareSummaryMarkdown(
+        { sha: currentSha.toString().trim() },
+        outputs,
+        { verbose: true },
+      ),
+    );
+
+    const failed = outputs.failed.length > 0;
+    if (failed) {
+      throw new Error('Some checks failed');
+    }
+  } catch (err) {
+    console.error(err);
+    process.exit(1);
+  }
+}

packages/repo-tools/src/commands/repo/schema/openapi/verify.ts

@@ -38,7 +38,6 @@ async function verify(directoryPath: string) {
     // Unable to find spec at path.
     return;
   }
-
   const yaml = YAML.load(await fs.readFile(openapiPath, 'utf8'));
   await Parser.validate(cloneDeep(yaml) as any);
 

packages/repo-tools/src/lib/openapi/optic/helpers.ts

@@ -0,0 +1,302 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* eslint-disable no-nested-ternary */
+
+import {
+  compareSpecs,
+  groupDiffsByEndpoint,
+  Severity,
+  getOperationsChangedLabel,
+  getOperationsChanged,
+} from '@useoptic/openapi-utilities';
+import { GroupedDiffs } from '@useoptic/openapi-utilities/build/openapi3/group-diff';
+
+/**
+ * The below code is copied from https://github.com/opticdev/optic/blob/main/projects/optic/src/commands/ci/comment/common.ts#L82 for use
+ *  with a security flow for forked repositories.
+ */
+
+type Comparison = {
+  groupedDiffs: ReturnType<typeof groupDiffsByEndpoint>;
+  results: Awaited<ReturnType<typeof compareSpecs>>['results'];
+};
+
+export type CiRunDetails = {
+  completed: {
+    warnings: string[];
+    apiName: string;
+    opticWebUrl?: string | null;
+    comparison: Comparison;
+    specUrl?: string | null;
+    capture?: any;
+  }[];
+  warning?: { apiName: string; warning: string }[];
+  failed: { apiName: string; error: string }[];
+  noop: { apiName: string }[];
+  severity: Severity;
+};
+
+const getChecksLabel = (
+  results: CiRunDetails['completed'][number]['comparison']['results'],
+  severity: Severity,
+) => {
+  const totalChecks = results.length;
+  let failingChecks = 0;
+  let exemptedFailingChecks = 0;
+
+  for (const result of results) {
+    if (result.passed) continue;
+    if (result.severity < severity) continue;
+    if (result.exempted) exemptedFailingChecks += 1;
+    else failingChecks += 1;
+  }
+
+  const exemptedChunk =
+    exemptedFailingChecks > 0 ? `, ${exemptedFailingChecks} exempted` : '';
+
+  return failingChecks > 0
+    ? `⚠️ **${failingChecks}**/**${totalChecks}** failed${exemptedChunk}`
+    : totalChecks > 0
+    ? `✅ **${totalChecks}** passed${exemptedChunk}`
+    : `ℹ️ No automated checks have run`;
+};
+
+function getOperationsText(
+  groupedDiffs: GroupedDiffs,
+  options: { webUrl?: string | null; verbose: boolean; labelJoiner?: string },
+) {
+  const ops = getOperationsChanged(groupedDiffs);
+
+  const operationsText = options.verbose
+    ? [
+        ...[...ops.added].map(o => `\`${o}\` (added)`),
+        ...[...ops.changed].map(o => `\`${o}\` (changed)`),
+        ...[...ops.removed].map(o => `\`${o}\` (removed)`),
+      ].join('\n')
+    : '';
+  return `${getOperationsChangedLabel(groupedDiffs)}
+    
+      ${operationsText}
+    `;
+}
+
+const getCaptureIssuesLabel = ({
+  unmatchedInteractions,
+  mismatchedEndpoints,
+}: {
+  unmatchedInteractions: number;
+  mismatchedEndpoints: number;
+}) => {
+  return [
+    ...(unmatchedInteractions
+      ? [
+          `🆕 ${unmatchedInteractions} undocumented path${
+            unmatchedInteractions > 1 ? 's' : ''
+          }`,
+        ]
+      : []),
+    ...(mismatchedEndpoints
+      ? [
+          `⚠️  ${mismatchedEndpoints} mismatch${
+            mismatchedEndpoints > 1 ? 'es' : ''
+          }`,
+        ]
+      : []),
+  ].join('\n');
+};
+
+const getBreakagesRow = (breakage: CiRunDetails['completed'][number]) => {
+  return `
+  - ${breakage.apiName}
+  ${breakage.comparison.results.map(
+    s => `
+    - ${s.where}
+      ${'```'}
+      ${s.error}
+      ${'```'}`,
+  )}`;
+};
+
+const addSummaryLine = (items: any[] | number | undefined, label: string) => {
+  const length = Array.isArray(items) ? items.length : items;
+  if (!length) return '';
+  let text = length === 1 ? `1 API` : `${length} APIs`;
+  text += ` had ${label}`;
+  return text;
+};
+
+export const generateCompareSummaryMarkdown = (
+  commit: { sha: string },
+  results: CiRunDetails,
+  options: { verbose: boolean },
+) => {
+  const anyCompletedHasWarning = results.completed.some(
+    s => s.warnings.length > 0,
+  );
+  const anyCompletedHasCapture = results.completed.some(s => s.capture);
+  if (
+    results.completed.length === 0 &&
+    results.failed.length === 0 &&
+    results.failed.length === 0
+  ) {
+    return `No API changes detected for commit (${commit.sha})`;
+  }
+  const breakages = results.completed
+    .filter(s => s.comparison.results.some(e => !e.passed))
+    .map(e => ({
+      ...e,
+      comparison: {
+        ...e.comparison,
+        results: e.comparison.results.filter(f => !f.passed),
+      },
+    }));
+  const successfullyCompletedCount =
+    results.completed.length - breakages.length;
+  return `### Summary for commit (${commit.sha})
+
+${addSummaryLine(results.noop, 'no changes')}
+
+${addSummaryLine(breakages.length, 'breaking changes')}
+
+${addSummaryLine(successfullyCompletedCount, 'non-breaking changes')}
+
+${addSummaryLine(results.warning, 'warnings')}
+
+${
+  results.completed.length > 0
+    ? `### APIs with Changes
+
+<table>
+<thead>
+<tr>
+<th>API</th>
+<th>Changes</th>
+<th>Rules</th>
+${anyCompletedHasWarning ? '<th>Warnings</th>' : ''}
+${anyCompletedHasCapture ? '<th>Tests</th>' : ''}
+</tr>
+</thead>
+<tbody>
+${results.completed
+  .map(
+    s =>
+      `<tr>
+<td>
+${s.apiName}
+</td>
+<td>
+${getOperationsText(s.comparison.groupedDiffs, {
+  webUrl: s.opticWebUrl,
+  verbose: options.verbose,
+  labelJoiner: ',\n',
+})}
+</td>
+<td>
+${getChecksLabel(s.comparison.results, results.severity)}
+</td>
+
+${anyCompletedHasWarning ? `<td>${s.warnings.join('\n')}</td>` : ''}
+
+${
+  anyCompletedHasCapture
+    ? `
+<td>
+${
+  s.capture
+    ? s.capture.success
+      ? s.capture.mismatchedEndpoints || s.capture.unmatchedInteractions
+        ? getCaptureIssuesLabel({
+            unmatchedInteractions: s.capture.unmatchedInteractions,
+            mismatchedEndpoints: s.capture.mismatchedEndpoints,
+          })
+        : `✅ ${s.capture.percentCovered}% coverage`
+      : '❌ Failed to run'
+    : ''
+}
+</td>
+`
+    : ''
+}
+</tr>`,
+  )
+  .join('\n')}
+</tbody>
+</table>`
+    : ''
+}
+
+${
+  results.failed.length > 0
+    ? `### APIs with Errors
+
+<table>
+<thead>
+<tr>
+<th>API</th>
+<th>Error</th>
+</tr>
+</thead>
+<tbody>
+${results.failed
+  .map(
+    s => `<tr>
+<td>${s.apiName}</td>
+<td>
+
+${'```'}
+${s.error}
+${'```'}
+
+</td>
+</tr>`,
+  )
+  .join('\n')}
+</tbody>
+</table>
+`
+    : ''
+}
+
+${
+  results.warning && results.warning.length
+    ? `
+### APIs with Warnings
+<table>
+<thead>
+<tr>
+<th>API</th>
+<th>Warning</th>
+</tr>
+</thead>
+<tbody>
+      ${results.warning
+        .map(e => `<tr><td>${e.apiName}</td><td>${e.warning}</td></tr>`)
+        .join('\n')}
+</tbody>
+</table>`
+    : ''
+}
+${
+  breakages.length > 0
+    ? `
+### Routes with Breakages
+
+${breakages.map(getBreakagesRow).join('\n')}
+`
+    : ''
+}
+`;
+};

plugins/catalog-backend/package.json

@@ -43,6 +43,7 @@
   "scripts": {
     "build": "backstage-cli package build",
     "clean": "backstage-cli package clean",
+    "diff": "backstage-repo-tools package schema openapi diff",
     "fuzz": "backstage-repo-tools package schema openapi fuzz --exclude-checks response_schema_conformance",
     "generate": "backstage-repo-tools package schema openapi generate --server --client-package packages/catalog-client",
     "lint": "backstage-cli package lint",

yarn.lock

@@ -7638,6 +7638,7 @@ __metadata:
     "@types/is-glob": ^4.0.2
     "@types/node": ^18.17.8
     "@types/prettier": ^2.0.0
+    "@useoptic/openapi-utilities": ^0.54.8
     chalk: ^4.0.0
     codeowners-utils: ^1.0.2
     command-exists: ^1.2.9
@@ -17339,6 +17340,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@useoptic/json-pointer-helpers@npm:0.54.8":
+  version: 0.54.8
+  resolution: "@useoptic/json-pointer-helpers@npm:0.54.8"
+  dependencies:
+    jsonpointer: ^5.0.1
+    minimatch: 9.0.3
+  checksum: 4eddabb6dce3ca8160dcd4904299b6964945c3fe47d39bfeca6c68b9a50b058b901a6fb10ab168295475d651df3349149faa5f27f77293e15b6eee8d4417432e
+  languageName: node
+  linkType: hard
+
 "@useoptic/openapi-io@npm:0.50.10":
   version: 0.50.10
   resolution: "@useoptic/openapi-io@npm:0.50.10"
@@ -17393,6 +17404,31 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@useoptic/openapi-utilities@npm:^0.54.8":
+  version: 0.54.8
+  resolution: "@useoptic/openapi-utilities@npm:0.54.8"
+  dependencies:
+    "@useoptic/json-pointer-helpers": 0.54.8
+    ajv: ^8.6.0
+    ajv-errors: ~3.0.0
+    ajv-formats: ~2.1.0
+    chalk: ^4.1.2
+    fast-deep-equal: ^3.1.3
+    is-url: ^1.2.4
+    js-yaml: ^4.1.0
+    json-stable-stringify: ^1.0.1
+    lodash.groupby: ^4.6.0
+    lodash.isequal: ^4.5.0
+    lodash.omit: ^4.5.0
+    node-machine-id: ^1.1.12
+    openapi-types: ^12.0.2
+    ts-invariant: ^0.9.3
+    url-join: ^4.0.1
+    yaml-ast-parser: ^0.0.43
+  checksum: fa9e9f430c77687591aaf8b43b7b31a7c2f80fe9c140aaa978f1948f84d3e974181c91c3d8ec3e06efca9735c7826290baf4be72063bf733887aa632b40c3c4a
+  languageName: node
+  linkType: hard
+
 "@useoptic/optic@npm:^0.50.10":
   version: 0.50.10
   resolution: "@useoptic/optic@npm:0.50.10"