diff --git a/.changeset/cyan-grapes-confess.md b/.changeset/cyan-grapes-confess.md new file mode 100644 index 0000000000..a113f91fd9 --- /dev/null +++ b/.changeset/cyan-grapes-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': patch +--- + +The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. diff --git a/.github/vale/config/vocabularies/Backstage/accept.txt b/.github/vale/config/vocabularies/Backstage/accept.txt index 091b999dd7..7dea58e627 100644 --- a/.github/vale/config/vocabularies/Backstage/accept.txt +++ b/.github/vale/config/vocabularies/Backstage/accept.txt @@ -276,6 +276,7 @@ nohoist nonces noop npm +nullable nunjucks nvarchar nvm diff --git a/package.json b/package.json index 07f3755e63..f4b44fdf52 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:all": "backstage-cli repo build --all", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs --exclude 'plugins/@(adr|adr-backend|adr-common|airbrake|airbrake-backend|allure|analytics-module-ga|analytics-module-ga4|analytics-module-newrelic-browser|apache-airflow|api-docs|api-docs-module-protoc-gen-doc|apollo-explorer|app-visualizer|azure-devops|azure-devops-backend|azure-devops-common|azure-sites|azure-sites-backend|azure-sites-common|badges|badges-backend|bazaar|bazaar-backend|bitbucket-cloud-common|bitrise|catalog-graph|catalog-graphql|catalog-import|catalog-unprocessed-entities|cicd-statistics|cicd-statistics-module-gitlab|circleci|cloudbuild|code-climate|code-coverage|code-coverage-backend|codescene|config-schema|cost-insights|cost-insights-common|dynatrace|entity-feedback|entity-feedback-backend|entity-feedback-common|entity-validation|example-todo-list|example-todo-list-backend|example-todo-list-common|firehydrant|fossa|gcalendar|gcp-projects|git-release-manager|github-actions|github-deployments|github-issues|github-pull-requests-board|gitops-profiles|gocd|graphiql|graphql-backend|graphql-voyager|ilert|jenkins|jenkins-backend|jenkins-common|kafka|kafka-backend|lighthouse|lighthouse-backend|lighthouse-common|linguist|linguist-backend|linguist-common|microsoft-calendar|newrelic|newrelic-dashboard|nomad|nomad-backend|octopus-deploy|opencost|pagerduty|periskop|periskop-backend|playlist|playlist-backend|playlist-common|proxy-backend|puppetdb|rollbar|rollbar-backend|sentry|shortcuts|splunk-on-call|stack-overflow|stack-overflow-backend|stackstorm|tech-radar|tech-radar-2|todo|todo-backend|xcmetrics)'", "build:api-reports": "yarn build:api-reports:only --tsc", - "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags", + "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --sql-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags", "build:backend": "yarn workspace example-backend build", "build:knip-reports": "backstage-repo-tools knip-reports", "build:plugins-report": "node ./scripts/build-plugins-report", diff --git a/packages/backend-defaults/report-auth.sql.md b/packages/backend-defaults/report-auth.sql.md new file mode 100644 index 0000000000..9551c34bb0 --- /dev/null +++ b/packages/backend-defaults/report-auth.sql.md @@ -0,0 +1,15 @@ +## SQL Report file for "@backstage/backend-defaults/auth" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `backstage_backend_public_keys__keys` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | ------------------- | -------- | ---------- | ------- | +| `expires_at` | `character varying` | false | 255 | - | +| `id` | `character varying` | false | 255 | - | +| `key` | `text` | false | - | - | + +### Indices + +- `backstage_backend_public_keys__keys_pkey` (`id`) unique primary diff --git a/packages/backend-defaults/report-scheduler.sql.md b/packages/backend-defaults/report-scheduler.sql.md new file mode 100644 index 0000000000..5542ac48b0 --- /dev/null +++ b/packages/backend-defaults/report-scheduler.sql.md @@ -0,0 +1,18 @@ +## SQL Report file for "@backstage/backend-defaults/scheduler" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `backstage_backend_tasks__tasks` + +| Column | Type | Nullable | Max Length | Default | +| ------------------------ | -------------------------- | -------- | ---------- | ------- | +| `current_run_expires_at` | `timestamp with time zone` | true | - | - | +| `current_run_started_at` | `timestamp with time zone` | true | - | - | +| `current_run_ticket` | `text` | true | - | - | +| `id` | `character varying` | false | 255 | - | +| `next_run_start_at` | `timestamp with time zone` | true | - | - | +| `settings_json` | `text` | false | - | - | + +### Indices + +- `backstage_backend_tasks__tasks_pkey` (`id`) unique primary diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index da483fe016..672c22944a 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -33,6 +33,7 @@ Options: --ci --tsc --docs + --sql-reports --include --exclude -a, --allow-warnings diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index b8223afe5c..f5c6a0c060 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -49,6 +49,7 @@ "@backstage/cli-node": "workspace:^", "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", + "@electric-sql/pglite": "^0.2.15", "@manypkg/get-packages": "^1.1.3", "@microsoft/api-documenter": "^7.25.7", "@microsoft/api-extractor": "^7.47.2", @@ -70,6 +71,9 @@ "glob": "^8.0.3", "is-glob": "^4.0.3", "js-yaml": "^4.1.0", + "just-diff": "^6.0.2", + "knex": "^3.0.0", + "knex-pglite": "^0.11.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", "p-limit": "^3.0.2", diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index c55410ce4a..0bac1cbeb1 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -284,7 +284,7 @@ export async function getTsDocConfig() { return tsdocConfigFile; } -function logApiReportInstructions() { +export function logApiReportInstructions() { console.log(''); console.log( '*************************************************************************************', @@ -398,6 +398,7 @@ export async function runApiExtraction({ filename => // https://regex101.com/r/QDZIV0/2 filename !== 'knip-report.md' && + !filename.endsWith('.sql.md') && // this has to temporarily match all old api report formats filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/), ), @@ -1218,6 +1219,7 @@ export async function categorizePackageDirs(packageDirs: string[]) { const dirs = packageDirs.slice(); const tsPackageDirs = new Array(); const cliPackageDirs = new Array(); + const sqlPackageDirs = new Array(); await Promise.all( Array(10) @@ -1241,6 +1243,11 @@ export async function categorizePackageDirs(packageDirs: string[]) { if (!role) { return; // Ignore packages without roles } + if ( + await fs.pathExists(cliPaths.resolveTargetRoot(dir, 'migrations')) + ) { + sqlPackageDirs.push(dir); + } // TODO(Rugvip): Inlined packages are ignored because we can't handle @internal exports // gracefully, and we don't want to have to mark all exports @public etc. // It would be good if we could include these packages though. @@ -1256,7 +1263,7 @@ export async function categorizePackageDirs(packageDirs: string[]) { }), ); - return { tsPackageDirs, cliPackageDirs }; + return { tsPackageDirs, cliPackageDirs, sqlPackageDirs }; } function parseHelpPage(helpPageContent: string) { diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts index 94ae42d715..02de908b8e 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -38,6 +38,7 @@ jest.mock('./api-extractor', () => ({ return { tsPackageDirs: p, cliPackageDirs: p, + sqlPackageDirs: [], }; }), runApiExtraction: jest.fn(), diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 4e568102d9..bec9cfd6c3 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -24,6 +24,7 @@ import { } from './api-extractor'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; +import { runSqlExtraction } from './sql-reports'; type Options = { ci?: boolean; @@ -79,9 +80,8 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { await generateTypeDeclarations(tsconfigFilePath); } - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( - selectedPackageDirs, - ); + const { tsPackageDirs, cliPackageDirs, sqlPackageDirs } = + await categorizePackageDirs(selectedPackageDirs); if (tsPackageDirs.length > 0) { console.log('# Generating package API reports'); @@ -104,6 +104,14 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { }); } + if (sqlPackageDirs.length > 0 && opts.sqlReports) { + console.log('# Generating package SQL reports'); + await runSqlExtraction({ + packageDirs: sqlPackageDirs, + isLocalBuild: !isCiBuild, + }); + } + if (isDocsBuild) { console.log('# Generating package documentation'); await buildDocs({ diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts new file mode 100644 index 0000000000..c804c96450 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts @@ -0,0 +1,98 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SchemaInfo } from './types'; + +function sortedEntries(obj: Record): [string, T][] { + return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)); +} + +function code(str: unknown): string { + if (str === '-') { + return str; + } + return `\`${str}\``; +} + +export function generateSqlReport(options: { + reportName: string; + failedDownMigration?: string; + schemaInfo: SchemaInfo; +}): string { + const { reportName, failedDownMigration, schemaInfo } = options; + + const output = [ + `## SQL Report file for "${reportName}"`, + '', + '> Do not edit this file. It is a report generated by `yarn build:api-reports`', + '', + ]; + + if (failedDownMigration) { + output.push('> [!WARNING]'); + output.push(`> Failed to migrate down from '${failedDownMigration}'`); + output.push(''); + } + + if (Object.keys(schemaInfo.sequences).length > 0) { + output.push('## Sequences'); + output.push(''); + for (const [sequenceName, sequenceInfo] of sortedEntries( + schemaInfo.sequences, + )) { + output.push(`- ${code(sequenceName)} (${sequenceInfo.type})`); + } + output.push(''); + } + + for (const [tableName, tableInfo] of sortedEntries(schemaInfo.tables)) { + output.push(`## Table ${code(tableName)}`); + output.push(''); + output.push(' | Column | Type | Nullable | Max Length | Default |'); + output.push(' |--------|------|----------|------------|---------|'); + for (const [columnName, columnInfo] of sortedEntries(tableInfo.columns)) { + output.push( + ` | ${code(columnName)} | ${code(columnInfo.type)} | ${ + columnInfo.nullable + } | ${columnInfo.maxLength ?? '-'} | ${code( + columnInfo.defaultValue ?? '-', + )} |`, + ); + } + output.push(''); + + if (Object.keys(tableInfo.indices).length > 0) { + output.push('### Indices'); + output.push(''); + for (const [indexName, indexInfo] of sortedEntries(tableInfo.indices)) { + const indexType = [ + indexInfo.unique && 'unique', + indexInfo.primary && 'primary', + ] + .filter(Boolean) + .join(' '); + output.push( + `- ${code(indexName)} (${indexInfo.columns.map(code).join(', ')})${ + indexType ? ` ${indexType}` : '' + }`, + ); + } + output.push(''); + } + } + + return output.join('\n'); +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts new file mode 100644 index 0000000000..fc0da743ae --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; +import { SchemaIndexInfo, SchemaInfo, SchemaSequenceInfo } from './types'; + +export async function getPgSchemaInfo(knex: Knex): Promise { + const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` + SELECT table_name as name + FROM information_schema.tables + WHERE + table_schema = 'public' + AND table_type = 'BASE TABLE' + AND table_name NOT LIKE 'knex_migrations%' + `); + + const tables = Object.fromEntries( + await Promise.all( + tableNames.map(async ({ name }) => { + const columns = await knex.table(name).columnInfo(); + const { rows: indices } = await knex.raw<{ + rows: SchemaIndexInfo[]; + }>( + ` + SELECT + index_class.relname as name, + index.indisunique as unique, + index.indisprimary as primary, + json_agg(attribute.attname ORDER BY keys.rn) as columns + FROM + pg_class table_class, + pg_class index_class, + pg_index index, + UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) + INNER JOIN pg_attribute attribute + ON attribute.attnum = keys.id + WHERE + table_class.oid = index.indrelid + AND table_class.relkind = 'r' + AND table_class.relname = ? + AND index_class.oid = index.indexrelid + AND attribute.attrelid = table_class.oid + GROUP BY index_class.relname, index.indexrelid + `, + [name], + ); + return [ + name, + { + name, + columns, + indices: Object.fromEntries( + indices.map(index => [index.name, index]), + ), + }, + ]; + }), + ), + ); + + const { rows: sequences } = await knex.raw<{ + rows: SchemaSequenceInfo[]; + }>(` + SELECT sequence_name as name, data_type as type + FROM information_schema.sequences + WHERE + sequence_schema = 'public' + AND sequence_name NOT LIKE 'knex_migrations%' + `); + + return { + tables, + sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), + }; +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts new file mode 100644 index 0000000000..d6b29d6f82 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runSqlExtraction } from './runSqlExtraction'; diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts new file mode 100644 index 0000000000..001c4c6c45 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -0,0 +1,212 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs, { readJson } from 'fs-extra'; +import { relative as relativePath } from 'path'; +import { paths as cliPaths } from '../../../lib/paths'; +import { diff as justDiff } from 'just-diff'; +import { SchemaInfo } from './types'; +import { getPgSchemaInfo } from './getPgSchemaInfo'; +import { generateSqlReport } from './generateSqlReport'; +import type { Knex } from 'knex'; +import { logApiReportInstructions } from '../api-extractor'; + +interface SqlExtractionOptions { + packageDirs: string[]; + isLocalBuild: boolean; +} + +export async function runSqlExtraction(options: SqlExtractionOptions) { + const { default: Knex } = await import('knex'); + const { default: ClientPgLite } = await import('knex-pglite'); + + // Since we're passing this as the client we need to replace the `config.client` with `pg` afterwards + class WrappedClientPgLite extends ClientPgLite { + constructor(config: any) { + super({ ...config, client: 'pg' }); + } + } + + let dbIndex = 1; + + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } + + const { name: pkgName } = await readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); + + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + const database = `extractor-${dbIndex++}`; + + const knex = Knex({ + client: WrappedClientPgLite, + dialect: 'postgres', + connection: { + database, + }, + }); + + await knex.raw('CREATE DATABASE ??', [database]); + + await runSingleSqlExtraction( + packageDir, + migrationTarget, + pkgName, + knex, + options, + ); + } + } +} + +async function runSingleSqlExtraction( + targetDir: string, + migrationTarget: string, + pkgName: string, + knex: Knex, + options: SqlExtractionOptions, +) { + const migrationDir = cliPaths.resolveTargetRoot( + targetDir, + 'migrations', + migrationTarget, + ); + + const reportName = + migrationTarget === '.' ? pkgName : `${pkgName}/${migrationTarget}`; + + console.log(`Generating SQL report for ${reportName}`); + + const migrationsListResult = await knex.migrate.list({ + directory: migrationDir, + }); + const migrations: string[] = migrationsListResult[1].map( + (m: { file: string }) => m.file, + ); + + const schemaInfoBeforeMigration = new Map(); + + for (const migration of migrations) { + const schemaInfo = await getPgSchemaInfo(knex); + schemaInfoBeforeMigration.set(migration, schemaInfo); + + await knex.migrate.up({ + directory: migrationDir, + name: migration, + }); + } + + const schemaInfo = await getPgSchemaInfo(knex); + + let failedDownMigration: string | undefined = undefined; + for (const migration of migrations.toReversed()) { + await knex.migrate.down({ + directory: migrationDir, + name: migration, + }); + const after = await getPgSchemaInfo(knex); + const before = schemaInfoBeforeMigration.get(migration); + if (!before) { + throw new Error(`No previous result for migration ${migration}`); + } + + const diff = justDiff(before, after); + if (diff.length !== 0) { + console.log( + `Migration ${migration} is not reversible: ${JSON.stringify( + diff, + null, + 2, + )}`, + ); + failedDownMigration = migration; + break; + } + } + + const report = prettyReport( + generateSqlReport({ + reportName, + failedDownMigration, + schemaInfo, + }), + ); + + const reportPath = cliPaths.resolveTargetRoot( + targetDir, + `report${migrationTarget === '.' ? '' : `-${migrationTarget}`}.sql.md`, + ); + const existingReport = await fs.readFile(reportPath, 'utf8').catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (existingReport !== report) { + if (options.isLocalBuild) { + console.warn(`SQL report changed for ${targetDir}`); + await fs.writeFile(reportPath, report); + } else { + logApiReportInstructions(); + + if (existingReport) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + cliPaths.targetRoot, + reportPath, + )}, expecting the following content:`, + ); + console.log(''); + + console.log(report); + + logApiReportInstructions(); + } + throw new Error(`Report ${reportPath} is out of date`); + } + } +} + +function prettyReport(content: string): string { + try { + const prettier = require('prettier') as typeof import('prettier'); + + const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {}; + return prettier.format(content, { + ...config, + parser: 'markdown', + }); + } catch (e) { + return content; + } +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts new file mode 100644 index 0000000000..c45d3fa01d --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; + +export type SchemaColumnInfo = { + name: string; + type: string; + nullable: boolean; + maxLength: number | null; + defaultValue: Knex.Value; +}; + +export type SchemaIndexInfo = { + name: string; + unique: boolean; + primary: boolean; + columns: string[]; +}; + +export type SchemaTableInfo = { + name: string; + columns: Record; + indices: Record; +}; + +export type SchemaSequenceInfo = { + name: string; + type: string; +}; + +export type SchemaInfo = { + tables: Record; + sequences: Record; +}; diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index e3c4c9e41f..4e984c72de 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -187,6 +187,7 @@ export function registerCommands(program: Command) { .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') + .option('--sql-reports', 'Also generate SQL reports from migration files') .option( '--include ', 'Only include packages matching the provided patterns', diff --git a/plugins/app-backend/report.sql.md b/plugins/app-backend/report.sql.md new file mode 100644 index 0000000000..e7bc9780fc --- /dev/null +++ b/plugins/app-backend/report.sql.md @@ -0,0 +1,17 @@ +## SQL Report file for "@backstage/plugin-app-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `static_assets_cache` + +| Column | Type | Nullable | Max Length | Default | +| ------------------ | -------------------------- | -------- | ---------- | ------------------------------ | +| `content` | `bytea` | false | - | - | +| `last_modified_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `namespace` | `character varying` | false | 255 | `'default'::character varying` | +| `path` | `text` | false | - | - | + +### Indices + +- `static_asset_cache_last_modified_at_idx` (`last_modified_at`) +- `static_assets_cache_pkey` (`namespace`, `path`) unique primary diff --git a/plugins/auth-backend/report.sql.md b/plugins/auth-backend/report.sql.md new file mode 100644 index 0000000000..7eef7a33cb --- /dev/null +++ b/plugins/auth-backend/report.sql.md @@ -0,0 +1,44 @@ +## SQL Report file for "@backstage/plugin-auth-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +> [!WARNING] +> Failed to migrate down from '20220321100910_timestamptz_again.js' + +## Table `sessions` + +| Column | Type | Nullable | Max Length | Default | +| --------- | -------------------------- | -------- | ---------- | ------- | +| `expired` | `timestamp with time zone` | false | - | - | +| `sess` | `text` | false | - | - | +| `sid` | `character varying` | false | 255 | - | + +### Indices + +- `sessions_expired_idx` (`expired`) +- `sessions_pkey` (`sid`) unique primary +- `sessions_sid_idx` (`sid`) + +## Table `signing_keys` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `key` | `text` | false | - | - | +| `kid` | `character varying` | false | 255 | - | + +### Indices + +- `signing_keys_pkey` (`kid`) unique primary + +## Table `user_info` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | -------------------------- | -------- | ---------- | ------- | +| `exp` | `timestamp with time zone` | false | - | - | +| `user_entity_ref` | `character varying` | false | 255 | - | +| `user_info` | `text` | false | - | - | + +### Indices + +- `user_info_pkey` (`user_entity_ref`) unique primary diff --git a/plugins/catalog-backend-module-incremental-ingestion/report.sql.md b/plugins/catalog-backend-module-incremental-ingestion/report.sql.md new file mode 100644 index 0000000000..d904d73e0d --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/report.sql.md @@ -0,0 +1,53 @@ +## SQL Report file for "@backstage/plugin-catalog-backend-module-incremental-ingestion" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `ingestion_mark_entities` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | ------------------- | -------- | ---------- | ------- | +| `id` | `uuid` | false | - | - | +| `ingestion_mark_id` | `uuid` | false | - | - | +| `ref` | `character varying` | false | 255 | - | + +### Indices + +- `ingestion_mark_entities_pkey` (`id`) unique primary +- `ingestion_mark_entity_ingestion_mark_id_idx` (`ingestion_mark_id`) + +## Table `ingestion_marks` + +| Column | Type | Nullable | Max Length | Default | +| -------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `cursor` | `json` | true | - | - | +| `id` | `uuid` | false | - | - | +| `ingestion_id` | `uuid` | false | - | - | +| `sequence` | `integer` | false | - | `0` | + +### Indices + +- `ingestion_mark_ingestion_id_idx` (`ingestion_id`) +- `ingestion_marks_pkey` (`id`) unique primary + +## Table `ingestions` + +| Column | Type | Nullable | Max Length | Default | +| ------------------------ | -------------------------- | -------- | ---------- | ------------------- | +| `attempts` | `integer` | false | - | `0` | +| `completion_ticket` | `character varying` | false | 255 | - | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `id` | `uuid` | false | - | - | +| `ingestion_completed_at` | `timestamp with time zone` | true | - | - | +| `last_error` | `character varying` | true | 255 | - | +| `next_action` | `character varying` | false | 255 | - | +| `next_action_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `provider_name` | `character varying` | false | 255 | - | +| `rest_completed_at` | `timestamp with time zone` | true | - | - | +| `status` | `character varying` | false | 255 | - | + +### Indices + +- `ingestion_composite_index` (`provider_name`, `completion_ticket`) unique +- `ingestion_provider_name_idx` (`provider_name`) +- `ingestions_pkey` (`id`) unique primary diff --git a/plugins/catalog-backend/report.sql.md b/plugins/catalog-backend/report.sql.md new file mode 100644 index 0000000000..3402b05063 --- /dev/null +++ b/plugins/catalog-backend/report.sql.md @@ -0,0 +1,137 @@ +## SQL Report file for "@backstage/plugin-catalog-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +> [!WARNING] +> Failed to migrate down from '20241003170511_alter_target_in_locations.js' + +## Sequences + +- `location_update_log_id_seq` (bigint) +- `refresh_state_references_id_seq` (integer) + +## Table `final_entities` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | -------------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | false | 255 | - | +| `entity_ref` | `character varying` | false | 255 | - | +| `final_entity` | `text` | true | - | - | +| `hash` | `character varying` | false | 255 | - | +| `last_updated_at` | `timestamp with time zone` | true | - | - | +| `stitch_ticket` | `text` | false | - | - | + +### Indices + +- `final_entities_entity_ref_uniq` (`entity_ref`) unique +- `final_entities_pkey` (`entity_id`) unique primary + +## Table `location_update_log` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------------------------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `entity_name` | `text` | true | - | - | +| `id` | `bigint` | false | - | `nextval('location_update_log_id_seq'::regclass)` | +| `location_id` | `uuid` | true | - | - | +| `message` | `text` | true | - | - | +| `status` | `text` | false | - | - | + +### Indices + +- `location_update_log_pkey` (`id`) unique primary +- `update_log_location_id_idx` (`location_id`) + +## Table `locations` + +| Column | Type | Nullable | Max Length | Default | +| -------- | ------------------- | -------- | ---------- | ------- | +| `id` | `uuid` | false | - | - | +| `target` | `text` | true | - | - | +| `type` | `character varying` | false | 255 | - | + +### Indices + +- `locations_pkey` (`id`) unique primary + +## Table `refresh_keys` + +| Column | Type | Nullable | Max Length | Default | +| ----------- | ------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | false | 255 | - | +| `key` | `character varying` | false | 255 | - | + +### Indices + +- `refresh_keys_entity_id_idx` (`entity_id`) +- `refresh_keys_key_idx` (`key`) + +## Table `refresh_state` + +| Column | Type | Nullable | Max Length | Default | +| -------------------- | -------------------------- | -------- | ---------- | ------- | +| `cache` | `text` | true | - | - | +| `entity_id` | `character varying` | false | 255 | - | +| `entity_ref` | `character varying` | false | 255 | - | +| `errors` | `text` | false | - | - | +| `last_discovery_at` | `timestamp with time zone` | false | - | - | +| `location_key` | `text` | true | - | - | +| `next_stitch_at` | `timestamp with time zone` | true | - | - | +| `next_stitch_ticket` | `character varying` | true | 255 | - | +| `next_update_at` | `timestamp with time zone` | false | - | - | +| `processed_entity` | `text` | true | - | - | +| `result_hash` | `text` | true | - | - | +| `unprocessed_entity` | `text` | false | - | - | +| `unprocessed_hash` | `text` | true | - | - | + +### Indices + +- `refresh_state_entity_ref_uniq` (`entity_ref`) unique +- `refresh_state_next_stitch_at_idx` (`next_stitch_at`) +- `refresh_state_next_update_at_idx` (`next_update_at`) +- `refresh_state_pkey` (`entity_id`) unique primary + +## Table `refresh_state_references` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | --------- | -------- | ---------- | ------------------------------------------------------ | +| `id` | `integer` | false | - | `nextval('refresh_state_references_id_seq'::regclass)` | +| `source_entity_ref` | `text` | true | - | - | +| `source_key` | `text` | true | - | - | +| `target_entity_ref` | `text` | false | - | - | + +### Indices + +- `refresh_state_references_pkey` (`id`) unique primary +- `refresh_state_references_source_entity_ref_idx` (`source_entity_ref`) +- `refresh_state_references_source_key_idx` (`source_key`) +- `refresh_state_references_target_entity_ref_idx` (`target_entity_ref`) + +## Table `relations` + +| Column | Type | Nullable | Max Length | Default | +| ----------------------- | ------------------- | -------- | ---------- | ------- | +| `originating_entity_id` | `character varying` | false | 255 | - | +| `source_entity_ref` | `character varying` | false | 255 | - | +| `target_entity_ref` | `character varying` | false | 255 | - | +| `type` | `character varying` | false | 255 | - | + +### Indices + +- `relations_source_entity_id_idx` (`originating_entity_id`) +- `relations_source_entity_ref_idx` (`source_entity_ref`) + +## Table `search` + +| Column | Type | Nullable | Max Length | Default | +| ---------------- | ------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | true | 255 | - | +| `key` | `character varying` | false | 255 | - | +| `original_value` | `character varying` | true | 255 | - | +| `value` | `character varying` | true | 255 | - | + +### Indices + +- `search_entity_id_idx` (`entity_id`) +- `search_key_original_value_idx` (`key`, `original_value`) +- `search_key_value_idx` (`key`, `value`) diff --git a/plugins/events-backend/report.sql.md b/plugins/events-backend/report.sql.md new file mode 100644 index 0000000000..117b41afd5 --- /dev/null +++ b/plugins/events-backend/report.sql.md @@ -0,0 +1,38 @@ +## SQL Report file for "@backstage/plugin-events-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Sequences + +- `event_bus_events_id_seq` (bigint) + +## Table `event_bus_events` + +| Column | Type | Nullable | Max Length | Default | +| ---------------------- | -------------------------- | -------- | ---------- | ---------------------------------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | false | - | - | +| `data_json` | `text` | false | - | - | +| `id` | `bigint` | false | - | `nextval('event_bus_events_id_seq'::regclass)` | +| `notified_subscribers` | `ARRAY` | true | - | - | +| `topic` | `text` | false | - | - | + +### Indices + +- `event_bus_events_pkey` (`id`) unique primary +- `event_bus_events_topic_idx` (`topic`) + +## Table `event_bus_subscriptions` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | false | - | - | +| `id` | `character varying` | false | 255 | - | +| `read_until` | `bigint` | false | - | - | +| `topics` | `ARRAY` | true | - | - | +| `updated_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | + +### Indices + +- `event_bus_subscriptions_pkey` (`id`) unique primary diff --git a/plugins/notifications-backend/report.sql.md b/plugins/notifications-backend/report.sql.md new file mode 100644 index 0000000000..6b9066e77c --- /dev/null +++ b/plugins/notifications-backend/report.sql.md @@ -0,0 +1,76 @@ +## SQL Report file for "@backstage/plugin-notifications-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `broadcast` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `description` | `text` | true | - | - | +| `icon` | `character varying` | true | 255 | - | +| `id` | `uuid` | false | - | - | +| `link` | `text` | true | - | - | +| `origin` | `character varying` | false | 255 | - | +| `scope` | `character varying` | true | 255 | - | +| `severity` | `character varying` | false | 8 | - | +| `title` | `character varying` | false | 255 | - | +| `topic` | `character varying` | true | 255 | - | +| `updated` | `timestamp with time zone` | true | - | - | + +### Indices + +- `broadcast_cope_origin_idx` (`scope`, `origin`) +- `broadcast_pkey` (`id`) unique primary + +## Table `broadcast_user_status` + +| Column | Type | Nullable | Max Length | Default | +| -------------- | -------------------------- | -------- | ---------- | ------- | +| `broadcast_id` | `uuid` | false | - | - | +| `read` | `timestamp with time zone` | true | - | - | +| `saved` | `timestamp with time zone` | true | - | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `broadcast_user_idx` (`broadcast_id`, `user`) unique + +## Table `notification` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `description` | `text` | true | - | - | +| `icon` | `character varying` | true | 255 | - | +| `id` | `uuid` | false | - | - | +| `link` | `text` | true | - | - | +| `origin` | `character varying` | false | 255 | - | +| `read` | `timestamp with time zone` | true | - | - | +| `saved` | `timestamp with time zone` | true | - | - | +| `scope` | `character varying` | true | 255 | - | +| `severity` | `character varying` | false | 8 | - | +| `title` | `character varying` | false | 255 | - | +| `topic` | `character varying` | true | 255 | - | +| `updated` | `timestamp with time zone` | true | - | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `notification_pkey` (`id`) unique primary +- `notification_scope_origin_idx` (`scope`, `origin`) +- `notification_user_idx` (`user`) + +## Table `user_settings` + +| Column | Type | Nullable | Max Length | Default | +| --------- | ------------------- | -------- | ---------- | ------- | +| `channel` | `character varying` | false | 255 | - | +| `enabled` | `boolean` | false | - | `true` | +| `origin` | `character varying` | false | 255 | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `user_settings_unique_idx` (`user`, `channel`, `origin`) unique +- `user_settings_user_idx` (`user`) diff --git a/plugins/scaffolder-backend/report.sql.md b/plugins/scaffolder-backend/report.sql.md new file mode 100644 index 0000000000..34c57266aa --- /dev/null +++ b/plugins/scaffolder-backend/report.sql.md @@ -0,0 +1,40 @@ +## SQL Report file for "@backstage/plugin-scaffolder-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Sequences + +- `task_events_id_seq` (bigint) + +## Table `task_events` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ----------------------------------------- | +| `body` | `text` | false | - | - | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `event_type` | `text` | false | - | - | +| `id` | `bigint` | false | - | `nextval('task_events_id_seq'::regclass)` | +| `task_id` | `uuid` | false | - | - | + +### Indices + +- `task_events_pkey` (`id`) unique primary +- `task_events_task_id_idx` (`task_id`) + +## Table `tasks` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | true | - | - | +| `id` | `uuid` | false | - | - | +| `last_heartbeat_at` | `timestamp with time zone` | true | - | - | +| `secrets` | `text` | true | - | - | +| `spec` | `text` | false | - | - | +| `state` | `text` | true | - | - | +| `status` | `text` | false | - | - | +| `workspace` | `bytea` | true | - | - | + +### Indices + +- `tasks_pkey` (`id`) unique primary diff --git a/plugins/search-backend-module-pg/report.sql.md b/plugins/search-backend-module-pg/report.sql.md new file mode 100644 index 0000000000..ebb5bad3e4 --- /dev/null +++ b/plugins/search-backend-module-pg/report.sql.md @@ -0,0 +1,19 @@ +## SQL Report file for "@backstage/plugin-search-backend-module-pg" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `documents` + +| Column | Type | Nullable | Max Length | Default | +| ---------- | ---------- | -------- | ---------- | ------- | +| `body` | `tsvector` | false | - | - | +| `document` | `jsonb` | false | - | - | +| `hash` | `bytea` | false | - | - | +| `type` | `text` | false | - | - | + +### Indices + +- `documents_body_index` (`body`) +- `documents_document_index` (`document`) +- `documents_pkey` (`hash`) unique primary +- `documents_type_index` (`type`) diff --git a/plugins/user-settings-backend/report.sql.md b/plugins/user-settings-backend/report.sql.md new file mode 100644 index 0000000000..bb312cf015 --- /dev/null +++ b/plugins/user-settings-backend/report.sql.md @@ -0,0 +1,16 @@ +## SQL Report file for "@backstage/plugin-user-settings-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `user_settings` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | ------------------- | -------- | ---------- | ------- | +| `bucket` | `character varying` | false | 255 | - | +| `key` | `character varying` | false | 255 | - | +| `user_entity_ref` | `character varying` | false | 255 | - | +| `value` | `text` | false | - | - | + +### Indices + +- `user_settings_pkey` (`user_entity_ref`, `bucket`, `key`) unique primary diff --git a/yarn.lock b/yarn.lock index 7a685bdb5a..4087ec2562 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8293,6 +8293,7 @@ __metadata: "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" + "@electric-sql/pglite": ^0.2.15 "@manypkg/get-packages": ^1.1.3 "@microsoft/api-documenter": ^7.25.7 "@microsoft/api-extractor": ^7.47.2 @@ -8317,6 +8318,9 @@ __metadata: glob: ^8.0.3 is-glob: ^4.0.3 js-yaml: ^4.1.0 + just-diff: ^6.0.2 + knex: ^3.0.0 + knex-pglite: ^0.11.0 lodash: ^4.17.21 minimatch: ^9.0.0 p-limit: ^3.0.2 @@ -8959,6 +8963,13 @@ __metadata: languageName: node linkType: hard +"@electric-sql/pglite@npm:^0.2.14, @electric-sql/pglite@npm:^0.2.15": + version: 0.2.15 + resolution: "@electric-sql/pglite@npm:0.2.15" + checksum: 714136449bdbef92ffc8475ba2237418bbdcef3bdbeb0eb43ab2e9f5fc4c2d9c67c41cbcd1e5ad7ba758d22f1bfb996a3f372745509845e55d9f69349f396aca + languageName: node + linkType: hard + "@emotion/babel-plugin@npm:^11.13.5": version: 11.13.5 resolution: "@emotion/babel-plugin@npm:11.13.5" @@ -33862,6 +33873,13 @@ __metadata: languageName: node linkType: hard +"just-diff@npm:^6.0.2": + version: 6.0.2 + resolution: "just-diff@npm:6.0.2" + checksum: 1a0c7524f640cb88ab013862733e710f840927834208fd3b85cbc5da2ced97acc75e7dcfe493268ac6a6514c51dd8624d2fd9d057050efba3c02b81a6dcb7ff9 + languageName: node + linkType: hard + "just-extend@npm:^6.2.0": version: 6.2.0 resolution: "just-extend@npm:6.2.0" @@ -33968,7 +33986,17 @@ __metadata: languageName: node linkType: hard -"knex@npm:3, knex@npm:^3.0.0": +"knex-pglite@npm:^0.11.0": + version: 0.11.0 + resolution: "knex-pglite@npm:0.11.0" + dependencies: + "@electric-sql/pglite": ^0.2.14 + knex: 3.1.0 + checksum: 680de87b9c567bfc3009c81e85584e4e634ede8b98f14acd4d4463e1eae084cdfc0eed05979db742ee2af1bd6f857bb23ca28c4175abdf27b3d6e3e2dfcba4df + languageName: node + linkType: hard + +"knex@npm:3, knex@npm:3.1.0, knex@npm:^3.0.0": version: 3.1.0 resolution: "knex@npm:3.1.0" dependencies: