Merge pull request #28315 from backstage/rugvip/sql-extractor

repo-tools: add SQL reports

.changeset/cyan-grapes-confess.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': patch
+---
+
+The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag.

.github/vale/config/vocabularies/Backstage/accept.txt

@@ -276,6 +276,7 @@ nohoist
 nonces
 noop
 npm
+nullable
 nunjucks
 nvarchar
 nvm

package.json

@@ -17,7 +17,7 @@
     "build:all": "backstage-cli repo build --all",
     "build:api-docs": "LANG=en_EN yarn build:api-reports --docs --exclude 'plugins/@(adr|adr-backend|adr-common|airbrake|airbrake-backend|allure|analytics-module-ga|analytics-module-ga4|analytics-module-newrelic-browser|apache-airflow|api-docs|api-docs-module-protoc-gen-doc|apollo-explorer|app-visualizer|azure-devops|azure-devops-backend|azure-devops-common|azure-sites|azure-sites-backend|azure-sites-common|badges|badges-backend|bazaar|bazaar-backend|bitbucket-cloud-common|bitrise|catalog-graph|catalog-graphql|catalog-import|catalog-unprocessed-entities|cicd-statistics|cicd-statistics-module-gitlab|circleci|cloudbuild|code-climate|code-coverage|code-coverage-backend|codescene|config-schema|cost-insights|cost-insights-common|dynatrace|entity-feedback|entity-feedback-backend|entity-feedback-common|entity-validation|example-todo-list|example-todo-list-backend|example-todo-list-common|firehydrant|fossa|gcalendar|gcp-projects|git-release-manager|github-actions|github-deployments|github-issues|github-pull-requests-board|gitops-profiles|gocd|graphiql|graphql-backend|graphql-voyager|ilert|jenkins|jenkins-backend|jenkins-common|kafka|kafka-backend|lighthouse|lighthouse-backend|lighthouse-common|linguist|linguist-backend|linguist-common|microsoft-calendar|newrelic|newrelic-dashboard|nomad|nomad-backend|octopus-deploy|opencost|pagerduty|periskop|periskop-backend|playlist|playlist-backend|playlist-common|proxy-backend|puppetdb|rollbar|rollbar-backend|sentry|shortcuts|splunk-on-call|stack-overflow|stack-overflow-backend|stackstorm|tech-radar|tech-radar-2|todo|todo-backend|xcmetrics)'",
     "build:api-reports": "yarn build:api-reports:only --tsc",
-    "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags",
+    "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --sql-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags",
     "build:backend": "yarn workspace example-backend build",
     "build:knip-reports": "backstage-repo-tools knip-reports",
     "build:plugins-report": "node ./scripts/build-plugins-report",

packages/backend-defaults/report-auth.sql.md

@@ -0,0 +1,15 @@
+## SQL Report file for "@backstage/backend-defaults/auth"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `backstage_backend_public_keys__keys`
+
+| Column       | Type                | Nullable | Max Length | Default |
+| ------------ | ------------------- | -------- | ---------- | ------- |
+| `expires_at` | `character varying` | false    | 255        | -       |
+| `id`         | `character varying` | false    | 255        | -       |
+| `key`        | `text`              | false    | -          | -       |
+
+### Indices
+
+- `backstage_backend_public_keys__keys_pkey` (`id`) unique primary

packages/backend-defaults/report-scheduler.sql.md

@@ -0,0 +1,18 @@
+## SQL Report file for "@backstage/backend-defaults/scheduler"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `backstage_backend_tasks__tasks`
+
+| Column                   | Type                       | Nullable | Max Length | Default |
+| ------------------------ | -------------------------- | -------- | ---------- | ------- |
+| `current_run_expires_at` | `timestamp with time zone` | true     | -          | -       |
+| `current_run_started_at` | `timestamp with time zone` | true     | -          | -       |
+| `current_run_ticket`     | `text`                     | true     | -          | -       |
+| `id`                     | `character varying`        | false    | 255        | -       |
+| `next_run_start_at`      | `timestamp with time zone` | true     | -          | -       |
+| `settings_json`          | `text`                     | false    | -          | -       |
+
+### Indices
+
+- `backstage_backend_tasks__tasks_pkey` (`id`) unique primary

packages/repo-tools/cli-report.md

@@ -33,6 +33,7 @@ Options:
   --ci
   --tsc
   --docs
+  --sql-reports
   --include <pattern>
   --exclude <pattern>
   -a, --allow-warnings <allowWarningsPaths>

packages/repo-tools/package.json

@@ -49,6 +49,7 @@
     "@backstage/cli-node": "workspace:^",
     "@backstage/config-loader": "workspace:^",
     "@backstage/errors": "workspace:^",
+    "@electric-sql/pglite": "^0.2.15",
     "@manypkg/get-packages": "^1.1.3",
     "@microsoft/api-documenter": "^7.25.7",
     "@microsoft/api-extractor": "^7.47.2",
@@ -70,6 +71,9 @@
     "glob": "^8.0.3",
     "is-glob": "^4.0.3",
     "js-yaml": "^4.1.0",
+    "just-diff": "^6.0.2",
+    "knex": "^3.0.0",
+    "knex-pglite": "^0.11.0",
     "lodash": "^4.17.21",
     "minimatch": "^9.0.0",
     "p-limit": "^3.0.2",

packages/repo-tools/src/commands/api-reports/api-extractor.ts

@@ -284,7 +284,7 @@ export async function getTsDocConfig() {
   return tsdocConfigFile;
 }
 
-function logApiReportInstructions() {
+export function logApiReportInstructions() {
   console.log('');
   console.log(
     '*************************************************************************************',
@@ -398,6 +398,7 @@ export async function runApiExtraction({
         filename =>
           // https://regex101.com/r/QDZIV0/2
           filename !== 'knip-report.md' &&
+          !filename.endsWith('.sql.md') &&
           // this has to temporarily match all old api report formats
           filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/),
       ),
@@ -1218,6 +1219,7 @@ export async function categorizePackageDirs(packageDirs: string[]) {
   const dirs = packageDirs.slice();
   const tsPackageDirs = new Array<string>();
   const cliPackageDirs = new Array<string>();
+  const sqlPackageDirs = new Array<string>();
 
   await Promise.all(
     Array(10)
@@ -1241,6 +1243,11 @@ export async function categorizePackageDirs(packageDirs: string[]) {
           if (!role) {
             return; // Ignore packages without roles
           }
+          if (
+            await fs.pathExists(cliPaths.resolveTargetRoot(dir, 'migrations'))
+          ) {
+            sqlPackageDirs.push(dir);
+          }
           // TODO(Rugvip): Inlined packages are ignored because we can't handle @internal exports
           //               gracefully, and we don't want to have to mark all exports @public etc.
           //               It would be good if we could include these packages though.
@@ -1256,7 +1263,7 @@ export async function categorizePackageDirs(packageDirs: string[]) {
       }),
   );
 
-  return { tsPackageDirs, cliPackageDirs };
+  return { tsPackageDirs, cliPackageDirs, sqlPackageDirs };
 }
 
 function parseHelpPage(helpPageContent: string) {

packages/repo-tools/src/commands/api-reports/api-reports.test.ts

@@ -38,6 +38,7 @@ jest.mock('./api-extractor', () => ({
     return {
       tsPackageDirs: p,
       cliPackageDirs: p,
+      sqlPackageDirs: [],
     };
   }),
   runApiExtraction: jest.fn(),

packages/repo-tools/src/commands/api-reports/api-reports.ts

@@ -24,6 +24,7 @@ import {
 } from './api-extractor';
 import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
 import { generateTypeDeclarations } from './generateTypeDeclarations';
+import { runSqlExtraction } from './sql-reports';
 
 type Options = {
   ci?: boolean;
@@ -79,9 +80,8 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
     await generateTypeDeclarations(tsconfigFilePath);
   }
 
-  const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs(
-    selectedPackageDirs,
-  );
+  const { tsPackageDirs, cliPackageDirs, sqlPackageDirs } =
+    await categorizePackageDirs(selectedPackageDirs);
 
   if (tsPackageDirs.length > 0) {
     console.log('# Generating package API reports');
@@ -104,6 +104,14 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
     });
   }
 
+  if (sqlPackageDirs.length > 0 && opts.sqlReports) {
+    console.log('# Generating package SQL reports');
+    await runSqlExtraction({
+      packageDirs: sqlPackageDirs,
+      isLocalBuild: !isCiBuild,
+    });
+  }
+
   if (isDocsBuild) {
     console.log('# Generating package documentation');
     await buildDocs({

packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SchemaInfo } from './types';
+
+function sortedEntries<T>(obj: Record<string, T>): [string, T][] {
+  return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b));
+}
+
+function code(str: unknown): string {
+  if (str === '-') {
+    return str;
+  }
+  return `\`${str}\``;
+}
+
+export function generateSqlReport(options: {
+  reportName: string;
+  failedDownMigration?: string;
+  schemaInfo: SchemaInfo;
+}): string {
+  const { reportName, failedDownMigration, schemaInfo } = options;
+
+  const output = [
+    `## SQL Report file for "${reportName}"`,
+    '',
+    '> Do not edit this file. It is a report generated by `yarn build:api-reports`',
+    '',
+  ];
+
+  if (failedDownMigration) {
+    output.push('> [!WARNING]');
+    output.push(`> Failed to migrate down from '${failedDownMigration}'`);
+    output.push('');
+  }
+
+  if (Object.keys(schemaInfo.sequences).length > 0) {
+    output.push('## Sequences');
+    output.push('');
+    for (const [sequenceName, sequenceInfo] of sortedEntries(
+      schemaInfo.sequences,
+    )) {
+      output.push(`- ${code(sequenceName)} (${sequenceInfo.type})`);
+    }
+    output.push('');
+  }
+
+  for (const [tableName, tableInfo] of sortedEntries(schemaInfo.tables)) {
+    output.push(`## Table ${code(tableName)}`);
+    output.push('');
+    output.push('  | Column | Type | Nullable | Max Length | Default |');
+    output.push('  |--------|------|----------|------------|---------|');
+    for (const [columnName, columnInfo] of sortedEntries(tableInfo.columns)) {
+      output.push(
+        `  | ${code(columnName)} | ${code(columnInfo.type)} | ${
+          columnInfo.nullable
+        } | ${columnInfo.maxLength ?? '-'} | ${code(
+          columnInfo.defaultValue ?? '-',
+        )} |`,
+      );
+    }
+    output.push('');
+
+    if (Object.keys(tableInfo.indices).length > 0) {
+      output.push('### Indices');
+      output.push('');
+      for (const [indexName, indexInfo] of sortedEntries(tableInfo.indices)) {
+        const indexType = [
+          indexInfo.unique && 'unique',
+          indexInfo.primary && 'primary',
+        ]
+          .filter(Boolean)
+          .join(' ');
+        output.push(
+          `- ${code(indexName)} (${indexInfo.columns.map(code).join(', ')})${
+            indexType ? ` ${indexType}` : ''
+          }`,
+        );
+      }
+      output.push('');
+    }
+  }
+
+  return output.join('\n');
+}

packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Knex } from 'knex';
+import { SchemaIndexInfo, SchemaInfo, SchemaSequenceInfo } from './types';
+
+export async function getPgSchemaInfo(knex: Knex): Promise<SchemaInfo> {
+  const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(`
+    SELECT table_name as name
+    FROM information_schema.tables
+    WHERE
+      table_schema = 'public'
+      AND table_type = 'BASE TABLE'
+      AND table_name NOT LIKE 'knex_migrations%'
+  `);
+
+  const tables = Object.fromEntries(
+    await Promise.all(
+      tableNames.map(async ({ name }) => {
+        const columns = await knex.table(name).columnInfo();
+        const { rows: indices } = await knex.raw<{
+          rows: SchemaIndexInfo[];
+        }>(
+          `
+          SELECT
+            index_class.relname as name,
+            index.indisunique as unique,
+            index.indisprimary as primary,
+            json_agg(attribute.attname ORDER BY keys.rn) as columns
+          FROM
+            pg_class table_class,
+            pg_class index_class,
+            pg_index index,
+            UNNEST(index.indkey) WITH ORDINALITY keys(id, rn)
+          INNER JOIN pg_attribute attribute
+            ON attribute.attnum = keys.id
+          WHERE
+            table_class.oid = index.indrelid
+            AND table_class.relkind = 'r'
+            AND table_class.relname = ?
+            AND index_class.oid = index.indexrelid
+            AND attribute.attrelid = table_class.oid
+          GROUP BY index_class.relname, index.indexrelid
+        `,
+          [name],
+        );
+        return [
+          name,
+          {
+            name,
+            columns,
+            indices: Object.fromEntries(
+              indices.map(index => [index.name, index]),
+            ),
+          },
+        ];
+      }),
+    ),
+  );
+
+  const { rows: sequences } = await knex.raw<{
+    rows: SchemaSequenceInfo[];
+  }>(`
+    SELECT sequence_name as name, data_type as type
+    FROM information_schema.sequences
+    WHERE
+      sequence_schema = 'public'
+      AND sequence_name NOT LIKE 'knex_migrations%'
+  `);
+
+  return {
+    tables,
+    sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])),
+  };
+}

packages/repo-tools/src/commands/api-reports/sql-reports/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { runSqlExtraction } from './runSqlExtraction';

packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs, { readJson } from 'fs-extra';
+import { relative as relativePath } from 'path';
+import { paths as cliPaths } from '../../../lib/paths';
+import { diff as justDiff } from 'just-diff';
+import { SchemaInfo } from './types';
+import { getPgSchemaInfo } from './getPgSchemaInfo';
+import { generateSqlReport } from './generateSqlReport';
+import type { Knex } from 'knex';
+import { logApiReportInstructions } from '../api-extractor';
+
+interface SqlExtractionOptions {
+  packageDirs: string[];
+  isLocalBuild: boolean;
+}
+
+export async function runSqlExtraction(options: SqlExtractionOptions) {
+  const { default: Knex } = await import('knex');
+  const { default: ClientPgLite } = await import('knex-pglite');
+
+  // Since we're passing this as the client we need to replace the `config.client` with `pg` afterwards
+  class WrappedClientPgLite extends ClientPgLite {
+    constructor(config: any) {
+      super({ ...config, client: 'pg' });
+    }
+  }
+
+  let dbIndex = 1;
+
+  for (const packageDir of options.packageDirs) {
+    const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations');
+    if (!(await fs.pathExists(migrationDir))) {
+      console.log(`No SQL migrations found in ${packageDir}`);
+      continue;
+    }
+
+    const { name: pkgName } = await readJson(
+      cliPaths.resolveTargetRoot(packageDir, 'package.json'),
+    );
+
+    const migrationFiles = await fs.readdir(migrationDir, {
+      withFileTypes: true,
+    });
+
+    const migrationTargets = migrationFiles
+      .filter(entry => entry.isDirectory())
+      .map(entry => entry.name);
+    if (migrationFiles.some(entry => entry.isFile())) {
+      migrationTargets.push('.');
+    }
+
+    for (const migrationTarget of migrationTargets) {
+      const database = `extractor-${dbIndex++}`;
+
+      const knex = Knex({
+        client: WrappedClientPgLite,
+        dialect: 'postgres',
+        connection: {
+          database,
+        },
+      });
+
+      await knex.raw('CREATE DATABASE ??', [database]);
+
+      await runSingleSqlExtraction(
+        packageDir,
+        migrationTarget,
+        pkgName,
+        knex,
+        options,
+      );
+    }
+  }
+}
+
+async function runSingleSqlExtraction(
+  targetDir: string,
+  migrationTarget: string,
+  pkgName: string,
+  knex: Knex,
+  options: SqlExtractionOptions,
+) {
+  const migrationDir = cliPaths.resolveTargetRoot(
+    targetDir,
+    'migrations',
+    migrationTarget,
+  );
+
+  const reportName =
+    migrationTarget === '.' ? pkgName : `${pkgName}/${migrationTarget}`;
+
+  console.log(`Generating SQL report for ${reportName}`);
+
+  const migrationsListResult = await knex.migrate.list({
+    directory: migrationDir,
+  });
+  const migrations: string[] = migrationsListResult[1].map(
+    (m: { file: string }) => m.file,
+  );
+
+  const schemaInfoBeforeMigration = new Map<string, SchemaInfo>();
+
+  for (const migration of migrations) {
+    const schemaInfo = await getPgSchemaInfo(knex);
+    schemaInfoBeforeMigration.set(migration, schemaInfo);
+
+    await knex.migrate.up({
+      directory: migrationDir,
+      name: migration,
+    });
+  }
+
+  const schemaInfo = await getPgSchemaInfo(knex);
+
+  let failedDownMigration: string | undefined = undefined;
+  for (const migration of migrations.toReversed()) {
+    await knex.migrate.down({
+      directory: migrationDir,
+      name: migration,
+    });
+    const after = await getPgSchemaInfo(knex);
+    const before = schemaInfoBeforeMigration.get(migration);
+    if (!before) {
+      throw new Error(`No previous result for migration ${migration}`);
+    }
+
+    const diff = justDiff(before, after);
+    if (diff.length !== 0) {
+      console.log(
+        `Migration ${migration} is not reversible: ${JSON.stringify(
+          diff,
+          null,
+          2,
+        )}`,
+      );
+      failedDownMigration = migration;
+      break;
+    }
+  }
+
+  const report = prettyReport(
+    generateSqlReport({
+      reportName,
+      failedDownMigration,
+      schemaInfo,
+    }),
+  );
+
+  const reportPath = cliPaths.resolveTargetRoot(
+    targetDir,
+    `report${migrationTarget === '.' ? '' : `-${migrationTarget}`}.sql.md`,
+  );
+  const existingReport = await fs.readFile(reportPath, 'utf8').catch(error => {
+    if (error.code === 'ENOENT') {
+      return undefined;
+    }
+    throw error;
+  });
+  if (existingReport !== report) {
+    if (options.isLocalBuild) {
+      console.warn(`SQL report changed for ${targetDir}`);
+      await fs.writeFile(reportPath, report);
+    } else {
+      logApiReportInstructions();
+
+      if (existingReport) {
+        console.log('');
+        console.log(
+          `The conflicting file is ${relativePath(
+            cliPaths.targetRoot,
+            reportPath,
+          )}, expecting the following content:`,
+        );
+        console.log('');
+
+        console.log(report);
+
+        logApiReportInstructions();
+      }
+      throw new Error(`Report ${reportPath} is out of date`);
+    }
+  }
+}
+
+function prettyReport(content: string): string {
+  try {
+    const prettier = require('prettier') as typeof import('prettier');
+
+    const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {};
+    return prettier.format(content, {
+      ...config,
+      parser: 'markdown',
+    });
+  } catch (e) {
+    return content;
+  }
+}

packages/repo-tools/src/commands/api-reports/sql-reports/types.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Knex } from 'knex';
+
+export type SchemaColumnInfo = {
+  name: string;
+  type: string;
+  nullable: boolean;
+  maxLength: number | null;
+  defaultValue: Knex.Value;
+};
+
+export type SchemaIndexInfo = {
+  name: string;
+  unique: boolean;
+  primary: boolean;
+  columns: string[];
+};
+
+export type SchemaTableInfo = {
+  name: string;
+  columns: Record<string, SchemaColumnInfo>;
+  indices: Record<string, SchemaIndexInfo>;
+};
+
+export type SchemaSequenceInfo = {
+  name: string;
+  type: string;
+};
+
+export type SchemaInfo = {
+  tables: Record<string, SchemaTableInfo>;
+  sequences: Record<string, SchemaSequenceInfo>;
+};

packages/repo-tools/src/commands/index.ts

@@ -187,6 +187,7 @@ export function registerCommands(program: Command) {
     .option('--ci', 'CI run checks that there is no changes on API reports')
     .option('--tsc', 'executes the tsc compilation before extracting the APIs')
     .option('--docs', 'generates the api documentation')
+    .option('--sql-reports', 'Also generate SQL reports from migration files')
     .option(
       '--include <pattern>',
       'Only include packages matching the provided patterns',

plugins/app-backend/report.sql.md

@@ -0,0 +1,17 @@
+## SQL Report file for "@backstage/plugin-app-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `static_assets_cache`
+
+| Column             | Type                       | Nullable | Max Length | Default                        |
+| ------------------ | -------------------------- | -------- | ---------- | ------------------------------ |
+| `content`          | `bytea`                    | false    | -          | -                              |
+| `last_modified_at` | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP`            |
+| `namespace`        | `character varying`        | false    | 255        | `'default'::character varying` |
+| `path`             | `text`                     | false    | -          | -                              |
+
+### Indices
+
+- `static_asset_cache_last_modified_at_idx` (`last_modified_at`)
+- `static_assets_cache_pkey` (`namespace`, `path`) unique primary

plugins/auth-backend/report.sql.md

@@ -0,0 +1,44 @@
+## SQL Report file for "@backstage/plugin-auth-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+> [!WARNING]
+> Failed to migrate down from '20220321100910_timestamptz_again.js'
+
+## Table `sessions`
+
+| Column    | Type                       | Nullable | Max Length | Default |
+| --------- | -------------------------- | -------- | ---------- | ------- |
+| `expired` | `timestamp with time zone` | false    | -          | -       |
+| `sess`    | `text`                     | false    | -          | -       |
+| `sid`     | `character varying`        | false    | 255        | -       |
+
+### Indices
+
+- `sessions_expired_idx` (`expired`)
+- `sessions_pkey` (`sid`) unique primary
+- `sessions_sid_idx` (`sid`)
+
+## Table `signing_keys`
+
+| Column       | Type                       | Nullable | Max Length | Default             |
+| ------------ | -------------------------- | -------- | ---------- | ------------------- |
+| `created_at` | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `key`        | `text`                     | false    | -          | -                   |
+| `kid`        | `character varying`        | false    | 255        | -                   |
+
+### Indices
+
+- `signing_keys_pkey` (`kid`) unique primary
+
+## Table `user_info`
+
+| Column            | Type                       | Nullable | Max Length | Default |
+| ----------------- | -------------------------- | -------- | ---------- | ------- |
+| `exp`             | `timestamp with time zone` | false    | -          | -       |
+| `user_entity_ref` | `character varying`        | false    | 255        | -       |
+| `user_info`       | `text`                     | false    | -          | -       |
+
+### Indices
+
+- `user_info_pkey` (`user_entity_ref`) unique primary

plugins/catalog-backend-module-incremental-ingestion/report.sql.md

@@ -0,0 +1,53 @@
+## SQL Report file for "@backstage/plugin-catalog-backend-module-incremental-ingestion"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `ingestion_mark_entities`
+
+| Column              | Type                | Nullable | Max Length | Default |
+| ------------------- | ------------------- | -------- | ---------- | ------- |
+| `id`                | `uuid`              | false    | -          | -       |
+| `ingestion_mark_id` | `uuid`              | false    | -          | -       |
+| `ref`               | `character varying` | false    | 255        | -       |
+
+### Indices
+
+- `ingestion_mark_entities_pkey` (`id`) unique primary
+- `ingestion_mark_entity_ingestion_mark_id_idx` (`ingestion_mark_id`)
+
+## Table `ingestion_marks`
+
+| Column         | Type                       | Nullable | Max Length | Default             |
+| -------------- | -------------------------- | -------- | ---------- | ------------------- |
+| `created_at`   | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `cursor`       | `json`                     | true     | -          | -                   |
+| `id`           | `uuid`                     | false    | -          | -                   |
+| `ingestion_id` | `uuid`                     | false    | -          | -                   |
+| `sequence`     | `integer`                  | false    | -          | `0`                 |
+
+### Indices
+
+- `ingestion_mark_ingestion_id_idx` (`ingestion_id`)
+- `ingestion_marks_pkey` (`id`) unique primary
+
+## Table `ingestions`
+
+| Column                   | Type                       | Nullable | Max Length | Default             |
+| ------------------------ | -------------------------- | -------- | ---------- | ------------------- |
+| `attempts`               | `integer`                  | false    | -          | `0`                 |
+| `completion_ticket`      | `character varying`        | false    | 255        | -                   |
+| `created_at`             | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `id`                     | `uuid`                     | false    | -          | -                   |
+| `ingestion_completed_at` | `timestamp with time zone` | true     | -          | -                   |
+| `last_error`             | `character varying`        | true     | 255        | -                   |
+| `next_action`            | `character varying`        | false    | 255        | -                   |
+| `next_action_at`         | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `provider_name`          | `character varying`        | false    | 255        | -                   |
+| `rest_completed_at`      | `timestamp with time zone` | true     | -          | -                   |
+| `status`                 | `character varying`        | false    | 255        | -                   |
+
+### Indices
+
+- `ingestion_composite_index` (`provider_name`, `completion_ticket`) unique
+- `ingestion_provider_name_idx` (`provider_name`)
+- `ingestions_pkey` (`id`) unique primary

plugins/catalog-backend/report.sql.md

@@ -0,0 +1,137 @@
+## SQL Report file for "@backstage/plugin-catalog-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+> [!WARNING]
+> Failed to migrate down from '20241003170511_alter_target_in_locations.js'
+
+## Sequences
+
+- `location_update_log_id_seq` (bigint)
+- `refresh_state_references_id_seq` (integer)
+
+## Table `final_entities`
+
+| Column            | Type                       | Nullable | Max Length | Default |
+| ----------------- | -------------------------- | -------- | ---------- | ------- |
+| `entity_id`       | `character varying`        | false    | 255        | -       |
+| `entity_ref`      | `character varying`        | false    | 255        | -       |
+| `final_entity`    | `text`                     | true     | -          | -       |
+| `hash`            | `character varying`        | false    | 255        | -       |
+| `last_updated_at` | `timestamp with time zone` | true     | -          | -       |
+| `stitch_ticket`   | `text`                     | false    | -          | -       |
+
+### Indices
+
+- `final_entities_entity_ref_uniq` (`entity_ref`) unique
+- `final_entities_pkey` (`entity_id`) unique primary
+
+## Table `location_update_log`
+
+| Column        | Type                       | Nullable | Max Length | Default                                           |
+| ------------- | -------------------------- | -------- | ---------- | ------------------------------------------------- |
+| `created_at`  | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP`                               |
+| `entity_name` | `text`                     | true     | -          | -                                                 |
+| `id`          | `bigint`                   | false    | -          | `nextval('location_update_log_id_seq'::regclass)` |
+| `location_id` | `uuid`                     | true     | -          | -                                                 |
+| `message`     | `text`                     | true     | -          | -                                                 |
+| `status`      | `text`                     | false    | -          | -                                                 |
+
+### Indices
+
+- `location_update_log_pkey` (`id`) unique primary
+- `update_log_location_id_idx` (`location_id`)
+
+## Table `locations`
+
+| Column   | Type                | Nullable | Max Length | Default |
+| -------- | ------------------- | -------- | ---------- | ------- |
+| `id`     | `uuid`              | false    | -          | -       |
+| `target` | `text`              | true     | -          | -       |
+| `type`   | `character varying` | false    | 255        | -       |
+
+### Indices
+
+- `locations_pkey` (`id`) unique primary
+
+## Table `refresh_keys`
+
+| Column      | Type                | Nullable | Max Length | Default |
+| ----------- | ------------------- | -------- | ---------- | ------- |
+| `entity_id` | `character varying` | false    | 255        | -       |
+| `key`       | `character varying` | false    | 255        | -       |
+
+### Indices
+
+- `refresh_keys_entity_id_idx` (`entity_id`)
+- `refresh_keys_key_idx` (`key`)
+
+## Table `refresh_state`
+
+| Column               | Type                       | Nullable | Max Length | Default |
+| -------------------- | -------------------------- | -------- | ---------- | ------- |
+| `cache`              | `text`                     | true     | -          | -       |
+| `entity_id`          | `character varying`        | false    | 255        | -       |
+| `entity_ref`         | `character varying`        | false    | 255        | -       |
+| `errors`             | `text`                     | false    | -          | -       |
+| `last_discovery_at`  | `timestamp with time zone` | false    | -          | -       |
+| `location_key`       | `text`                     | true     | -          | -       |
+| `next_stitch_at`     | `timestamp with time zone` | true     | -          | -       |
+| `next_stitch_ticket` | `character varying`        | true     | 255        | -       |
+| `next_update_at`     | `timestamp with time zone` | false    | -          | -       |
+| `processed_entity`   | `text`                     | true     | -          | -       |
+| `result_hash`        | `text`                     | true     | -          | -       |
+| `unprocessed_entity` | `text`                     | false    | -          | -       |
+| `unprocessed_hash`   | `text`                     | true     | -          | -       |
+
+### Indices
+
+- `refresh_state_entity_ref_uniq` (`entity_ref`) unique
+- `refresh_state_next_stitch_at_idx` (`next_stitch_at`)
+- `refresh_state_next_update_at_idx` (`next_update_at`)
+- `refresh_state_pkey` (`entity_id`) unique primary
+
+## Table `refresh_state_references`
+
+| Column              | Type      | Nullable | Max Length | Default                                                |
+| ------------------- | --------- | -------- | ---------- | ------------------------------------------------------ |
+| `id`                | `integer` | false    | -          | `nextval('refresh_state_references_id_seq'::regclass)` |
+| `source_entity_ref` | `text`    | true     | -          | -                                                      |
+| `source_key`        | `text`    | true     | -          | -                                                      |
+| `target_entity_ref` | `text`    | false    | -          | -                                                      |
+
+### Indices
+
+- `refresh_state_references_pkey` (`id`) unique primary
+- `refresh_state_references_source_entity_ref_idx` (`source_entity_ref`)
+- `refresh_state_references_source_key_idx` (`source_key`)
+- `refresh_state_references_target_entity_ref_idx` (`target_entity_ref`)
+
+## Table `relations`
+
+| Column                  | Type                | Nullable | Max Length | Default |
+| ----------------------- | ------------------- | -------- | ---------- | ------- |
+| `originating_entity_id` | `character varying` | false    | 255        | -       |
+| `source_entity_ref`     | `character varying` | false    | 255        | -       |
+| `target_entity_ref`     | `character varying` | false    | 255        | -       |
+| `type`                  | `character varying` | false    | 255        | -       |
+
+### Indices
+
+- `relations_source_entity_id_idx` (`originating_entity_id`)
+- `relations_source_entity_ref_idx` (`source_entity_ref`)
+
+## Table `search`
+
+| Column           | Type                | Nullable | Max Length | Default |
+| ---------------- | ------------------- | -------- | ---------- | ------- |
+| `entity_id`      | `character varying` | true     | 255        | -       |
+| `key`            | `character varying` | false    | 255        | -       |
+| `original_value` | `character varying` | true     | 255        | -       |
+| `value`          | `character varying` | true     | 255        | -       |
+
+### Indices
+
+- `search_entity_id_idx` (`entity_id`)
+- `search_key_original_value_idx` (`key`, `original_value`)
+- `search_key_value_idx` (`key`, `value`)

plugins/events-backend/report.sql.md

@@ -0,0 +1,38 @@
+## SQL Report file for "@backstage/plugin-events-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Sequences
+
+- `event_bus_events_id_seq` (bigint)
+
+## Table `event_bus_events`
+
+| Column                 | Type                       | Nullable | Max Length | Default                                        |
+| ---------------------- | -------------------------- | -------- | ---------- | ---------------------------------------------- |
+| `created_at`           | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP`                            |
+| `created_by`           | `text`                     | false    | -          | -                                              |
+| `data_json`            | `text`                     | false    | -          | -                                              |
+| `id`                   | `bigint`                   | false    | -          | `nextval('event_bus_events_id_seq'::regclass)` |
+| `notified_subscribers` | `ARRAY`                    | true     | -          | -                                              |
+| `topic`                | `text`                     | false    | -          | -                                              |
+
+### Indices
+
+- `event_bus_events_pkey` (`id`) unique primary
+- `event_bus_events_topic_idx` (`topic`)
+
+## Table `event_bus_subscriptions`
+
+| Column       | Type                       | Nullable | Max Length | Default             |
+| ------------ | -------------------------- | -------- | ---------- | ------------------- |
+| `created_at` | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `created_by` | `text`                     | false    | -          | -                   |
+| `id`         | `character varying`        | false    | 255        | -                   |
+| `read_until` | `bigint`                   | false    | -          | -                   |
+| `topics`     | `ARRAY`                    | true     | -          | -                   |
+| `updated_at` | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+
+### Indices
+
+- `event_bus_subscriptions_pkey` (`id`) unique primary

plugins/notifications-backend/report.sql.md

@@ -0,0 +1,76 @@
+## SQL Report file for "@backstage/plugin-notifications-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `broadcast`
+
+| Column        | Type                       | Nullable | Max Length | Default             |
+| ------------- | -------------------------- | -------- | ---------- | ------------------- |
+| `created`     | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `description` | `text`                     | true     | -          | -                   |
+| `icon`        | `character varying`        | true     | 255        | -                   |
+| `id`          | `uuid`                     | false    | -          | -                   |
+| `link`        | `text`                     | true     | -          | -                   |
+| `origin`      | `character varying`        | false    | 255        | -                   |
+| `scope`       | `character varying`        | true     | 255        | -                   |
+| `severity`    | `character varying`        | false    | 8          | -                   |
+| `title`       | `character varying`        | false    | 255        | -                   |
+| `topic`       | `character varying`        | true     | 255        | -                   |
+| `updated`     | `timestamp with time zone` | true     | -          | -                   |
+
+### Indices
+
+- `broadcast_cope_origin_idx` (`scope`, `origin`)
+- `broadcast_pkey` (`id`) unique primary
+
+## Table `broadcast_user_status`
+
+| Column         | Type                       | Nullable | Max Length | Default |
+| -------------- | -------------------------- | -------- | ---------- | ------- |
+| `broadcast_id` | `uuid`                     | false    | -          | -       |
+| `read`         | `timestamp with time zone` | true     | -          | -       |
+| `saved`        | `timestamp with time zone` | true     | -          | -       |
+| `user`         | `character varying`        | false    | 255        | -       |
+
+### Indices
+
+- `broadcast_user_idx` (`broadcast_id`, `user`) unique
+
+## Table `notification`
+
+| Column        | Type                       | Nullable | Max Length | Default             |
+| ------------- | -------------------------- | -------- | ---------- | ------------------- |
+| `created`     | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `description` | `text`                     | true     | -          | -                   |
+| `icon`        | `character varying`        | true     | 255        | -                   |
+| `id`          | `uuid`                     | false    | -          | -                   |
+| `link`        | `text`                     | true     | -          | -                   |
+| `origin`      | `character varying`        | false    | 255        | -                   |
+| `read`        | `timestamp with time zone` | true     | -          | -                   |
+| `saved`       | `timestamp with time zone` | true     | -          | -                   |
+| `scope`       | `character varying`        | true     | 255        | -                   |
+| `severity`    | `character varying`        | false    | 8          | -                   |
+| `title`       | `character varying`        | false    | 255        | -                   |
+| `topic`       | `character varying`        | true     | 255        | -                   |
+| `updated`     | `timestamp with time zone` | true     | -          | -                   |
+| `user`        | `character varying`        | false    | 255        | -                   |
+
+### Indices
+
+- `notification_pkey` (`id`) unique primary
+- `notification_scope_origin_idx` (`scope`, `origin`)
+- `notification_user_idx` (`user`)
+
+## Table `user_settings`
+
+| Column    | Type                | Nullable | Max Length | Default |
+| --------- | ------------------- | -------- | ---------- | ------- |
+| `channel` | `character varying` | false    | 255        | -       |
+| `enabled` | `boolean`           | false    | -          | `true`  |
+| `origin`  | `character varying` | false    | 255        | -       |
+| `user`    | `character varying` | false    | 255        | -       |
+
+### Indices
+
+- `user_settings_unique_idx` (`user`, `channel`, `origin`) unique
+- `user_settings_user_idx` (`user`)

plugins/scaffolder-backend/report.sql.md

@@ -0,0 +1,40 @@
+## SQL Report file for "@backstage/plugin-scaffolder-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Sequences
+
+- `task_events_id_seq` (bigint)
+
+## Table `task_events`
+
+| Column       | Type                       | Nullable | Max Length | Default                                   |
+| ------------ | -------------------------- | -------- | ---------- | ----------------------------------------- |
+| `body`       | `text`                     | false    | -          | -                                         |
+| `created_at` | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP`                       |
+| `event_type` | `text`                     | false    | -          | -                                         |
+| `id`         | `bigint`                   | false    | -          | `nextval('task_events_id_seq'::regclass)` |
+| `task_id`    | `uuid`                     | false    | -          | -                                         |
+
+### Indices
+
+- `task_events_pkey` (`id`) unique primary
+- `task_events_task_id_idx` (`task_id`)
+
+## Table `tasks`
+
+| Column              | Type                       | Nullable | Max Length | Default             |
+| ------------------- | -------------------------- | -------- | ---------- | ------------------- |
+| `created_at`        | `timestamp with time zone` | false    | -          | `CURRENT_TIMESTAMP` |
+| `created_by`        | `text`                     | true     | -          | -                   |
+| `id`                | `uuid`                     | false    | -          | -                   |
+| `last_heartbeat_at` | `timestamp with time zone` | true     | -          | -                   |
+| `secrets`           | `text`                     | true     | -          | -                   |
+| `spec`              | `text`                     | false    | -          | -                   |
+| `state`             | `text`                     | true     | -          | -                   |
+| `status`            | `text`                     | false    | -          | -                   |
+| `workspace`         | `bytea`                    | true     | -          | -                   |
+
+### Indices
+
+- `tasks_pkey` (`id`) unique primary

plugins/search-backend-module-pg/report.sql.md

@@ -0,0 +1,19 @@
+## SQL Report file for "@backstage/plugin-search-backend-module-pg"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `documents`
+
+| Column     | Type       | Nullable | Max Length | Default |
+| ---------- | ---------- | -------- | ---------- | ------- |
+| `body`     | `tsvector` | false    | -          | -       |
+| `document` | `jsonb`    | false    | -          | -       |
+| `hash`     | `bytea`    | false    | -          | -       |
+| `type`     | `text`     | false    | -          | -       |
+
+### Indices
+
+- `documents_body_index` (`body`)
+- `documents_document_index` (`document`)
+- `documents_pkey` (`hash`) unique primary
+- `documents_type_index` (`type`)

plugins/user-settings-backend/report.sql.md

@@ -0,0 +1,16 @@
+## SQL Report file for "@backstage/plugin-user-settings-backend"
+
+> Do not edit this file. It is a report generated by `yarn build:api-reports`
+
+## Table `user_settings`
+
+| Column            | Type                | Nullable | Max Length | Default |
+| ----------------- | ------------------- | -------- | ---------- | ------- |
+| `bucket`          | `character varying` | false    | 255        | -       |
+| `key`             | `character varying` | false    | 255        | -       |
+| `user_entity_ref` | `character varying` | false    | 255        | -       |
+| `value`           | `text`              | false    | -          | -       |
+
+### Indices
+
+- `user_settings_pkey` (`user_entity_ref`, `bucket`, `key`) unique primary

yarn.lock

@@ -8293,6 +8293,7 @@ __metadata:
     "@backstage/config-loader": "workspace:^"
     "@backstage/errors": "workspace:^"
     "@backstage/types": "workspace:^"
+    "@electric-sql/pglite": ^0.2.15
     "@manypkg/get-packages": ^1.1.3
     "@microsoft/api-documenter": ^7.25.7
     "@microsoft/api-extractor": ^7.47.2
@@ -8317,6 +8318,9 @@ __metadata:
     glob: ^8.0.3
     is-glob: ^4.0.3
     js-yaml: ^4.1.0
+    just-diff: ^6.0.2
+    knex: ^3.0.0
+    knex-pglite: ^0.11.0
     lodash: ^4.17.21
     minimatch: ^9.0.0
     p-limit: ^3.0.2
@@ -8959,6 +8963,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@electric-sql/pglite@npm:^0.2.14, @electric-sql/pglite@npm:^0.2.15":
+  version: 0.2.15
+  resolution: "@electric-sql/pglite@npm:0.2.15"
+  checksum: 714136449bdbef92ffc8475ba2237418bbdcef3bdbeb0eb43ab2e9f5fc4c2d9c67c41cbcd1e5ad7ba758d22f1bfb996a3f372745509845e55d9f69349f396aca
+  languageName: node
+  linkType: hard
+
 "@emotion/babel-plugin@npm:^11.13.5":
   version: 11.13.5
   resolution: "@emotion/babel-plugin@npm:11.13.5"
@@ -33862,6 +33873,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"just-diff@npm:^6.0.2":
+  version: 6.0.2
+  resolution: "just-diff@npm:6.0.2"
+  checksum: 1a0c7524f640cb88ab013862733e710f840927834208fd3b85cbc5da2ced97acc75e7dcfe493268ac6a6514c51dd8624d2fd9d057050efba3c02b81a6dcb7ff9
+  languageName: node
+  linkType: hard
+
 "just-extend@npm:^6.2.0":
   version: 6.2.0
   resolution: "just-extend@npm:6.2.0"
@@ -33968,7 +33986,17 @@ __metadata:
   languageName: node
   linkType: hard
 
-"knex@npm:3, knex@npm:^3.0.0":
+"knex-pglite@npm:^0.11.0":
+  version: 0.11.0
+  resolution: "knex-pglite@npm:0.11.0"
+  dependencies:
+    "@electric-sql/pglite": ^0.2.14
+    knex: 3.1.0
+  checksum: 680de87b9c567bfc3009c81e85584e4e634ede8b98f14acd4d4463e1eae084cdfc0eed05979db742ee2af1bd6f857bb23ca28c4175abdf27b3d6e3e2dfcba4df
+  languageName: node
+  linkType: hard
+
+"knex@npm:3, knex@npm:3.1.0, knex@npm:^3.0.0":
   version: 3.1.0
   resolution: "knex@npm:3.1.0"
   dependencies: