From e7272f9dfc54731e8f644750a527a6fd33989d6e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Dec 2024 11:47:41 +0100 Subject: [PATCH 01/19] repo-tools: initial SQL extraction wiring Signed-off-by: Patrik Oldsberg --- packages/repo-tools/package.json | 8 +- .../src/commands/api-reports/api-extractor.ts | 8 +- .../src/commands/api-reports/api-reports.ts | 14 ++- .../src/commands/api-reports/sql-report.ts | 70 +++++++++++++ yarn.lock | 98 ++++++++++++++++++- 5 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-report.ts diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index b8223afe5c..e7778ae2a9 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -70,6 +70,7 @@ "glob": "^8.0.3", "is-glob": "^4.0.3", "js-yaml": "^4.1.0", + "knex": "^3.0.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", "p-limit": "^3.0.2", @@ -84,17 +85,22 @@ "@backstage/types": "workspace:^", "@types/is-glob": "^4.0.2", "@types/node": "^20.16.0", - "@types/prettier": "^2.0.0" + "@types/prettier": "^2.0.0", + "embedded-postgres": "17.2.0-beta.15" }, "peerDependencies": { "@microsoft/api-extractor-model": "*", "@microsoft/tsdoc": "*", "@microsoft/tsdoc-config": "*", "@useoptic/optic": "^1.0.0", + "embedded-postgres": "17.2.0-beta.15", "prettier": "^2.8.1", "typescript": "> 3.0.0" }, "peerDependenciesMeta": { + "embedded-postgres": { + "optional": true + }, "prettier": { "optional": true } diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index c55410ce4a..dde3a33b54 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -1218,6 +1218,7 @@ export async function categorizePackageDirs(packageDirs: string[]) { const dirs = packageDirs.slice(); const tsPackageDirs = new Array(); const cliPackageDirs = new Array(); + const sqlPackageDirs = new Array(); await Promise.all( Array(10) @@ -1241,6 +1242,11 @@ export async function categorizePackageDirs(packageDirs: string[]) { if (!role) { return; // Ignore packages without roles } + if ( + await fs.pathExists(cliPaths.resolveTargetRoot(dir, 'migrations')) + ) { + sqlPackageDirs.push(dir); + } // TODO(Rugvip): Inlined packages are ignored because we can't handle @internal exports // gracefully, and we don't want to have to mark all exports @public etc. // It would be good if we could include these packages though. @@ -1256,7 +1262,7 @@ export async function categorizePackageDirs(packageDirs: string[]) { }), ); - return { tsPackageDirs, cliPackageDirs }; + return { tsPackageDirs, cliPackageDirs, sqlPackageDirs }; } function parseHelpPage(helpPageContent: string) { diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 4e568102d9..debe22724d 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -24,6 +24,7 @@ import { } from './api-extractor'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; +import { runSqlExtraction } from './sql-report'; type Options = { ci?: boolean; @@ -79,9 +80,8 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { await generateTypeDeclarations(tsconfigFilePath); } - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( - selectedPackageDirs, - ); + const { tsPackageDirs, cliPackageDirs, sqlPackageDirs } = + await categorizePackageDirs(selectedPackageDirs); if (tsPackageDirs.length > 0) { console.log('# Generating package API reports'); @@ -104,6 +104,14 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { }); } + if (sqlPackageDirs.length > 0) { + console.log('# Generating package SQL reports'); + await runSqlExtraction({ + packageDirs: sqlPackageDirs, + isLocalBuild: !isCiBuild, + }); + } + if (isDocsBuild) { console.log('# Generating package documentation'); await buildDocs({ diff --git a/packages/repo-tools/src/commands/api-reports/sql-report.ts b/packages/repo-tools/src/commands/api-reports/sql-report.ts new file mode 100644 index 0000000000..670f2933e0 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-report.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { paths as cliPaths } from '../../lib/paths'; + +interface SqlExtractionOptions { + packageDirs: string[]; + isLocalBuild: boolean; +} + +export async function runSqlExtraction(options: SqlExtractionOptions) { + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } + + console.log(`Extracting SQL migrations from ${packageDir}`); + + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + await runSingleSqlExtraction(packageDir, migrationTarget, options); + } + } +} + +async function runSingleSqlExtraction( + targetDir: string, + migrationPath: string, + options: SqlExtractionOptions, +) { + const migrationDir = cliPaths.resolveTargetRoot( + targetDir, + 'migrations', + migrationPath, + ); + + console.log(`Extracting SQL from ${migrationDir}`); + + const knex = await import('knex'); + const EmbeddedPostgres = await import('embedded-postgres'); + + console.log(`DEBUG: knex=`, knex); + console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); +} diff --git a/yarn.lock b/yarn.lock index 4c87e9fb4b..00a5807990 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8313,10 +8313,12 @@ __metadata: codeowners-utils: ^1.0.2 command-exists: ^1.2.9 commander: ^12.0.0 + embedded-postgres: 17.2.0-beta.15 fs-extra: ^11.2.0 glob: ^8.0.3 is-glob: ^4.0.3 js-yaml: ^4.1.0 + knex: ^3.0.0 lodash: ^4.17.21 minimatch: ^9.0.0 p-limit: ^3.0.2 @@ -8329,9 +8331,12 @@ __metadata: "@microsoft/tsdoc": "*" "@microsoft/tsdoc-config": "*" "@useoptic/optic": ^1.0.0 + embedded-postgres: 17.2.0-beta.15 prettier: ^2.8.1 typescript: "> 3.0.0" peerDependenciesMeta: + embedded-postgres: + optional: true prettier: optional: true bin: @@ -8958,6 +8963,62 @@ __metadata: languageName: node linkType: hard +"@embedded-postgres/darwin-arm64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/darwin-arm64@npm:17.2.0-beta.15" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@embedded-postgres/darwin-x64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/darwin-x64@npm:17.2.0-beta.15" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-arm64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/linux-arm64@npm:17.2.0-beta.15" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-arm@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/linux-arm@npm:17.2.0-beta.15" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@embedded-postgres/linux-ia32@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/linux-ia32@npm:17.2.0-beta.15" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + +"@embedded-postgres/linux-ppc64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/linux-ppc64@npm:17.2.0-beta.15" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-x64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/linux-x64@npm:17.2.0-beta.15" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@embedded-postgres/windows-x64@npm:^17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "@embedded-postgres/windows-x64@npm:17.2.0-beta.15" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@emotion/babel-plugin@npm:^11.13.5": version: 11.13.5 resolution: "@emotion/babel-plugin@npm:11.13.5" @@ -26836,6 +26897,41 @@ __metadata: languageName: node linkType: hard +"embedded-postgres@npm:17.2.0-beta.15": + version: 17.2.0-beta.15 + resolution: "embedded-postgres@npm:17.2.0-beta.15" + dependencies: + "@embedded-postgres/darwin-arm64": ^17.2.0-beta.15 + "@embedded-postgres/darwin-x64": ^17.2.0-beta.15 + "@embedded-postgres/linux-arm": ^17.2.0-beta.15 + "@embedded-postgres/linux-arm64": ^17.2.0-beta.15 + "@embedded-postgres/linux-ia32": ^17.2.0-beta.15 + "@embedded-postgres/linux-ppc64": ^17.2.0-beta.15 + "@embedded-postgres/linux-x64": ^17.2.0-beta.15 + "@embedded-postgres/windows-x64": ^17.2.0-beta.15 + async-exit-hook: ^2.0.1 + pg: ^8.7.3 + dependenciesMeta: + "@embedded-postgres/darwin-arm64": + optional: true + "@embedded-postgres/darwin-x64": + optional: true + "@embedded-postgres/linux-arm": + optional: true + "@embedded-postgres/linux-arm64": + optional: true + "@embedded-postgres/linux-ia32": + optional: true + "@embedded-postgres/linux-ppc64": + optional: true + "@embedded-postgres/linux-x64": + optional: true + "@embedded-postgres/windows-x64": + optional: true + checksum: 53b261693dcc83cea6cc2589794dbc69bcf4cae508f2021d4686d7fb08e686e93db97392ebd28dfd2bb0126060b54fa609487d9929a4fee19da6d66dc568863f + languageName: node + linkType: hard + "emittery@npm:^0.13.1": version: 0.13.1 resolution: "emittery@npm:0.13.1" @@ -38383,7 +38479,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.3, pg@npm:^8.9.0": +"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": version: 8.13.1 resolution: "pg@npm:8.13.1" dependencies: From a985fd265ff731c2d721d13bb38111dadaca3798 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Dec 2024 13:06:29 +0100 Subject: [PATCH 02/19] repo-tools: sql extraction schema info Signed-off-by: Patrik Oldsberg --- packages/repo-tools/package.json | 1 + .../src/commands/api-reports/sql-report.ts | 202 +++++++++++++++++- yarn.lock | 8 + 3 files changed, 208 insertions(+), 3 deletions(-) diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index e7778ae2a9..b5f1bbc596 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -70,6 +70,7 @@ "glob": "^8.0.3", "is-glob": "^4.0.3", "js-yaml": "^4.1.0", + "just-diff": "^6.0.2", "knex": "^3.0.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", diff --git a/packages/repo-tools/src/commands/api-reports/sql-report.ts b/packages/repo-tools/src/commands/api-reports/sql-report.ts index 670f2933e0..a4571f04ca 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-report.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-report.ts @@ -16,6 +16,9 @@ import fs from 'fs-extra'; import { paths as cliPaths } from '../../lib/paths'; +import { getPortPromise } from 'portfinder'; +import type { Knex } from 'knex'; +import { diff as justDiff } from 'just-diff'; interface SqlExtractionOptions { packageDirs: string[]; @@ -62,9 +65,202 @@ async function runSingleSqlExtraction( console.log(`Extracting SQL from ${migrationDir}`); - const knex = await import('knex'); - const EmbeddedPostgres = await import('embedded-postgres'); + const { default: Knex } = await import('knex'); + const { default: EmbeddedPostgres } = (await import( + 'embedded-postgres' + )) as typeof import('embedded-postgres/dist/index.d.ts'); - console.log(`DEBUG: knex=`, knex); + console.log(`DEBUG: knex=`, Knex); console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); + + const port = await getPortPromise({ + /* startPort: 5433, stopPort: 6543 */ + }); + console.log(`DEBUG: port=`, port); + + const pg = new EmbeddedPostgres({ + databaseDir: './data/db', + user: 'postgres', + password: 'password', + port, + persistent: false, + onError(_messageOrError) { + // console.error('EmbeddedPostgres error:', messageOrError); + }, + onLog(_message) { + // console.log('EmbeddedPostgres log:', message); + }, + }); + + // Create the cluster config files + await pg.initialise(); + + // Start the server + await pg.start(); + + await pg.createDatabase('extractor'); + + const knex = Knex({ + client: 'pg', + connection: { + host: 'localhost', + port, + user: 'postgres', + password: 'password', + database: 'extractor', + }, + }); + + const migrationsListResult = await knex.migrate.list({ + directory: migrationDir, + }); + const migrations: string[] = migrationsListResult[1].map( + (m: { file: string }) => m.file, + ); + + const schemaInfoBeforeMigration = new Map(); + + for (const migration of migrations) { + console.log(`DEBUG: UP ${migration}`); + const schemaInfo = await getPostgresSchemaInfo(knex); + schemaInfoBeforeMigration.set(migration, schemaInfo); + + await knex.migrate.up({ + directory: migrationDir, + name: migration, + }); + } + + const schemaInfo = await getPostgresSchemaInfo(knex); + console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2)); + + for (const migration of migrations.toReversed()) { + console.log(`DEBUG: DOWN ${migration}`); + await knex.migrate.down({ + directory: migrationDir, + name: migration, + }); + const after = await getPostgresSchemaInfo(knex); + const before = schemaInfoBeforeMigration.get(migration); + if (!before) { + throw new Error(`No previous result for migration ${migration}`); + } + + const diff = justDiff(before, after); + console.log(`DEBUG: diff=`, diff); + if (diff.length !== 0) { + console.log(`Migration ${migration} is not reversible`); + await pg.stop(); + return; + } + } + + // Stop the server + await pg.stop(); +} + +type SchemaColumnInfo = { + name: string; + type: string; + nullable: boolean; + maxLength: number | null; + defaultValue: Knex.Value; +}; + +type SchemaIndexInfo = { + name: string; + unique: boolean; + primary: boolean; + columns: string[]; +}; + +type SchemaTableInfo = { + name: string; + columns: Record; + indices: Record; +}; + +type SchemaSequenceInfo = { + name: string; + type: string; +}; + +type SchemaInfo = { + tables: Record; + sequences: Record; +}; + +async function getPostgresSchemaInfo(knex: Knex): Promise { + const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` + SELECT table_name as name + FROM information_schema.tables + WHERE + table_schema = 'public' + AND table_type = 'BASE TABLE' + AND table_name NOT LIKE 'knex_migrations%' + `); + + const tables = Object.fromEntries( + await Promise.all( + tableNames.map(async ({ name }) => { + const columns = await knex.table(name).columnInfo(); + const { rows: indices } = await knex.raw<{ + rows: SchemaIndexInfo[]; + }>( + ` + SELECT + index_class.relname as name, + index.indisunique as unique, + index.indisprimary as primary, + json_agg(attribute.attname ORDER BY keys.rn) as columns + FROM + pg_class table_class, + pg_class index_class, + pg_index index, + UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) + INNER JOIN pg_attribute attribute + ON attribute.attnum = keys.id + WHERE + table_class.oid = index.indrelid + AND table_class.relkind = 'r' + AND table_class.relname = ? + AND index_class.oid = index.indexrelid + AND attribute.attrelid = table_class.oid + GROUP BY index_class.relname, index.indexrelid + `, + [name], + ); + return [ + name, + { + name, + columns: Object.entries(columns).map( + ([columnName, columnInfo]) => ({ + ...columnInfo, + name: columnName, + }), + ), + indices: Object.fromEntries( + indices.map(index => [index.name, index]), + ), + }, + ]; + }), + ), + ); + + const { rows: sequences } = await knex.raw<{ + rows: SchemaSequenceInfo[]; + }>(` + SELECT sequence_name as name, data_type as type + FROM information_schema.sequences + WHERE + sequence_schema = 'public' + AND sequence_name NOT LIKE 'knex_migrations%' + `); + + return { + tables, + sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), + }; } diff --git a/yarn.lock b/yarn.lock index 00a5807990..0c67429722 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8318,6 +8318,7 @@ __metadata: glob: ^8.0.3 is-glob: ^4.0.3 js-yaml: ^4.1.0 + just-diff: ^6.0.2 knex: ^3.0.0 lodash: ^4.17.21 minimatch: ^9.0.0 @@ -33693,6 +33694,13 @@ __metadata: languageName: node linkType: hard +"just-diff@npm:^6.0.2": + version: 6.0.2 + resolution: "just-diff@npm:6.0.2" + checksum: 1a0c7524f640cb88ab013862733e710f840927834208fd3b85cbc5da2ced97acc75e7dcfe493268ac6a6514c51dd8624d2fd9d057050efba3c02b81a6dcb7ff9 + languageName: node + linkType: hard + "just-extend@npm:^6.2.0": version: 6.2.0 resolution: "just-extend@npm:6.2.0" From 175983bbda6bc166f9eccba30523265c92952080 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Dec 2024 13:21:22 +0100 Subject: [PATCH 03/19] repo-tools: move sql-reports to separate dir Signed-off-by: Patrik Oldsberg --- .../src/commands/api-reports/api-reports.ts | 2 +- .../commands/api-reports/sql-reports/index.ts | 17 +++++++++++++++++ .../runSqlExtraction.ts} | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/index.ts rename packages/repo-tools/src/commands/api-reports/{sql-report.ts => sql-reports/runSqlExtraction.ts} (99%) diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index debe22724d..1e10ade111 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -24,7 +24,7 @@ import { } from './api-extractor'; import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths'; import { generateTypeDeclarations } from './generateTypeDeclarations'; -import { runSqlExtraction } from './sql-report'; +import { runSqlExtraction } from './sql-reports'; type Options = { ci?: boolean; diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts new file mode 100644 index 0000000000..d6b29d6f82 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runSqlExtraction } from './runSqlExtraction'; diff --git a/packages/repo-tools/src/commands/api-reports/sql-report.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts similarity index 99% rename from packages/repo-tools/src/commands/api-reports/sql-report.ts rename to packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index a4571f04ca..6fcac744a8 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-report.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -15,7 +15,7 @@ */ import fs from 'fs-extra'; -import { paths as cliPaths } from '../../lib/paths'; +import { paths as cliPaths } from '../../../lib/paths'; import { getPortPromise } from 'portfinder'; import type { Knex } from 'knex'; import { diff as justDiff } from 'just-diff'; From be2fa90773b94381ebb02fc7e97ad83b601883ec Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Dec 2024 13:33:05 +0100 Subject: [PATCH 04/19] repo-tools: refactor sql-reports into split modules Signed-off-by: Patrik Oldsberg --- .../sql-reports/getPgSchemaInfo.ts | 93 ++++++++++++++ .../sql-reports/runSqlExtraction.ts | 119 +----------------- .../commands/api-reports/sql-reports/types.ts | 48 +++++++ packages/repo-tools/src/types.d.ts | 20 +++ 4 files changed, 167 insertions(+), 113 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/types.ts create mode 100644 packages/repo-tools/src/types.d.ts diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts new file mode 100644 index 0000000000..cc8db86ccd --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; +import { SchemaIndexInfo, SchemaInfo, SchemaSequenceInfo } from './types'; + +export async function getPgSchemaInfo(knex: Knex): Promise { + const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` + SELECT table_name as name + FROM information_schema.tables + WHERE + table_schema = 'public' + AND table_type = 'BASE TABLE' + AND table_name NOT LIKE 'knex_migrations%' + `); + + const tables = Object.fromEntries( + await Promise.all( + tableNames.map(async ({ name }) => { + const columns = await knex.table(name).columnInfo(); + const { rows: indices } = await knex.raw<{ + rows: SchemaIndexInfo[]; + }>( + ` + SELECT + index_class.relname as name, + index.indisunique as unique, + index.indisprimary as primary, + json_agg(attribute.attname ORDER BY keys.rn) as columns + FROM + pg_class table_class, + pg_class index_class, + pg_index index, + UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) + INNER JOIN pg_attribute attribute + ON attribute.attnum = keys.id + WHERE + table_class.oid = index.indrelid + AND table_class.relkind = 'r' + AND table_class.relname = ? + AND index_class.oid = index.indexrelid + AND attribute.attrelid = table_class.oid + GROUP BY index_class.relname, index.indexrelid + `, + [name], + ); + return [ + name, + { + name, + columns: Object.entries(columns).map( + ([columnName, columnInfo]) => ({ + ...columnInfo, + name: columnName, + }), + ), + indices: Object.fromEntries( + indices.map(index => [index.name, index]), + ), + }, + ]; + }), + ), + ); + + const { rows: sequences } = await knex.raw<{ + rows: SchemaSequenceInfo[]; + }>(` + SELECT sequence_name as name, data_type as type + FROM information_schema.sequences + WHERE + sequence_schema = 'public' + AND sequence_name NOT LIKE 'knex_migrations%' + `); + + return { + tables, + sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), + }; +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 6fcac744a8..2300fee301 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -17,8 +17,9 @@ import fs from 'fs-extra'; import { paths as cliPaths } from '../../../lib/paths'; import { getPortPromise } from 'portfinder'; -import type { Knex } from 'knex'; import { diff as justDiff } from 'just-diff'; +import { SchemaInfo } from './types'; +import { getPgSchemaInfo } from './getPgSchemaInfo'; interface SqlExtractionOptions { packageDirs: string[]; @@ -66,9 +67,7 @@ async function runSingleSqlExtraction( console.log(`Extracting SQL from ${migrationDir}`); const { default: Knex } = await import('knex'); - const { default: EmbeddedPostgres } = (await import( - 'embedded-postgres' - )) as typeof import('embedded-postgres/dist/index.d.ts'); + const { default: EmbeddedPostgres } = await import('embedded-postgres'); console.log(`DEBUG: knex=`, Knex); console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); @@ -122,7 +121,7 @@ async function runSingleSqlExtraction( for (const migration of migrations) { console.log(`DEBUG: UP ${migration}`); - const schemaInfo = await getPostgresSchemaInfo(knex); + const schemaInfo = await getPgSchemaInfo(knex); schemaInfoBeforeMigration.set(migration, schemaInfo); await knex.migrate.up({ @@ -131,7 +130,7 @@ async function runSingleSqlExtraction( }); } - const schemaInfo = await getPostgresSchemaInfo(knex); + const schemaInfo = await getPgSchemaInfo(knex); console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2)); for (const migration of migrations.toReversed()) { @@ -140,7 +139,7 @@ async function runSingleSqlExtraction( directory: migrationDir, name: migration, }); - const after = await getPostgresSchemaInfo(knex); + const after = await getPgSchemaInfo(knex); const before = schemaInfoBeforeMigration.get(migration); if (!before) { throw new Error(`No previous result for migration ${migration}`); @@ -158,109 +157,3 @@ async function runSingleSqlExtraction( // Stop the server await pg.stop(); } - -type SchemaColumnInfo = { - name: string; - type: string; - nullable: boolean; - maxLength: number | null; - defaultValue: Knex.Value; -}; - -type SchemaIndexInfo = { - name: string; - unique: boolean; - primary: boolean; - columns: string[]; -}; - -type SchemaTableInfo = { - name: string; - columns: Record; - indices: Record; -}; - -type SchemaSequenceInfo = { - name: string; - type: string; -}; - -type SchemaInfo = { - tables: Record; - sequences: Record; -}; - -async function getPostgresSchemaInfo(knex: Knex): Promise { - const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` - SELECT table_name as name - FROM information_schema.tables - WHERE - table_schema = 'public' - AND table_type = 'BASE TABLE' - AND table_name NOT LIKE 'knex_migrations%' - `); - - const tables = Object.fromEntries( - await Promise.all( - tableNames.map(async ({ name }) => { - const columns = await knex.table(name).columnInfo(); - const { rows: indices } = await knex.raw<{ - rows: SchemaIndexInfo[]; - }>( - ` - SELECT - index_class.relname as name, - index.indisunique as unique, - index.indisprimary as primary, - json_agg(attribute.attname ORDER BY keys.rn) as columns - FROM - pg_class table_class, - pg_class index_class, - pg_index index, - UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) - INNER JOIN pg_attribute attribute - ON attribute.attnum = keys.id - WHERE - table_class.oid = index.indrelid - AND table_class.relkind = 'r' - AND table_class.relname = ? - AND index_class.oid = index.indexrelid - AND attribute.attrelid = table_class.oid - GROUP BY index_class.relname, index.indexrelid - `, - [name], - ); - return [ - name, - { - name, - columns: Object.entries(columns).map( - ([columnName, columnInfo]) => ({ - ...columnInfo, - name: columnName, - }), - ), - indices: Object.fromEntries( - indices.map(index => [index.name, index]), - ), - }, - ]; - }), - ), - ); - - const { rows: sequences } = await knex.raw<{ - rows: SchemaSequenceInfo[]; - }>(` - SELECT sequence_name as name, data_type as type - FROM information_schema.sequences - WHERE - sequence_schema = 'public' - AND sequence_name NOT LIKE 'knex_migrations%' - `); - - return { - tables, - sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), - }; -} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts new file mode 100644 index 0000000000..c45d3fa01d --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; + +export type SchemaColumnInfo = { + name: string; + type: string; + nullable: boolean; + maxLength: number | null; + defaultValue: Knex.Value; +}; + +export type SchemaIndexInfo = { + name: string; + unique: boolean; + primary: boolean; + columns: string[]; +}; + +export type SchemaTableInfo = { + name: string; + columns: Record; + indices: Record; +}; + +export type SchemaSequenceInfo = { + name: string; + type: string; +}; + +export type SchemaInfo = { + tables: Record; + sequences: Record; +}; diff --git a/packages/repo-tools/src/types.d.ts b/packages/repo-tools/src/types.d.ts new file mode 100644 index 0000000000..276b6ec8f9 --- /dev/null +++ b/packages/repo-tools/src/types.d.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// It's missing a types entry point, but has types in dist +declare module 'embedded-postgres' { + export { default } from 'embedded-postgres/dist/index.d.ts'; +} From dee5cf2f3c49ef3e565a8ea4c7c42a17980f1f1c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:25:48 +0100 Subject: [PATCH 05/19] repo-tools: full SQL report implementation Signed-off-by: Patrik Oldsberg --- .../src/commands/api-reports/api-extractor.ts | 2 +- .../sql-reports/generateSqlReport.ts | 87 ++++++++ .../sql-reports/getPgSchemaInfo.ts | 7 +- .../sql-reports/runSqlExtraction.ts | 199 ++++++++++++------ 4 files changed, 219 insertions(+), 76 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index dde3a33b54..4db8077501 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -284,7 +284,7 @@ export async function getTsDocConfig() { return tsdocConfigFile; } -function logApiReportInstructions() { +export function logApiReportInstructions() { console.log(''); console.log( '*************************************************************************************', diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts new file mode 100644 index 0000000000..53165590eb --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SchemaInfo } from './types'; + +export function generateSqlReport(options: { + reportName: string; + failedDownMigration?: string; + schemaInfo: SchemaInfo; +}): string { + const { reportName, failedDownMigration, schemaInfo } = options; + + const output = [ + `## SQL Report file for "${reportName}"`, + '', + '> Do not edit this file. It is a report generated by `yarn build:api-reports`', + '', + ]; + + if (failedDownMigration) { + output.push('> [!WARNING]'); + output.push(`> Failed to migrate down from '${failedDownMigration}'`); + output.push(''); + } + + if (Object.keys(schemaInfo.sequences).length > 0) { + output.push('## Sequences'); + output.push(''); + for (const [sequenceName, sequenceInfo] of Object.entries( + schemaInfo.sequences, + )) { + output.push(`- \`${sequenceName}\` (${sequenceInfo.type})`); + } + output.push(''); + } + + for (const [tableName, tableInfo] of Object.entries(schemaInfo.tables)) { + output.push(`## Table \`${tableName}\``); + output.push(''); + output.push(' | Column | Type | Nullable | Max Length | Default |'); + output.push(' |--------|------|----------|------------|---------|'); + for (const [columnName, columnInfo] of Object.entries(tableInfo.columns)) { + output.push( + ` | \`${columnName}\` | ${columnInfo.type} | ${ + columnInfo.nullable + } | ${columnInfo.maxLength ?? '-'} | ${ + columnInfo.defaultValue ?? '-' + } |`, + ); + } + output.push(''); + + if (Object.keys(tableInfo.indices).length > 0) { + output.push('### Indices'); + output.push(''); + for (const [indexName, indexInfo] of Object.entries(tableInfo.indices)) { + const indexType = [ + indexInfo.unique && 'unique', + indexInfo.primary && 'primary', + ] + .filter(Boolean) + .join(' '); + output.push( + `- \`${indexName}\` (\`${indexInfo.columns.join('`, `')}\`)${ + indexType ? ` ${indexType}` : '' + }`, + ); + } + output.push(''); + } + } + + return output.join('\n'); +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts index cc8db86ccd..fc0da743ae 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts @@ -61,12 +61,7 @@ export async function getPgSchemaInfo(knex: Knex): Promise { name, { name, - columns: Object.entries(columns).map( - ([columnName, columnInfo]) => ({ - ...columnInfo, - name: columnName, - }), - ), + columns, indices: Object.fromEntries( indices.map(index => [index.name, index]), ), diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 2300fee301..9a78907cfd 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -14,12 +14,16 @@ * limitations under the License. */ -import fs from 'fs-extra'; +import fs, { readJson } from 'fs-extra'; +import { relative as relativePath } from 'path'; import { paths as cliPaths } from '../../../lib/paths'; import { getPortPromise } from 'portfinder'; import { diff as justDiff } from 'just-diff'; import { SchemaInfo } from './types'; import { getPgSchemaInfo } from './getPgSchemaInfo'; +import { generateSqlReport } from './generateSqlReport'; +import type { Knex } from 'knex'; +import { logApiReportInstructions } from '../api-extractor'; interface SqlExtractionOptions { packageDirs: string[]; @@ -27,60 +31,19 @@ interface SqlExtractionOptions { } export async function runSqlExtraction(options: SqlExtractionOptions) { - for (const packageDir of options.packageDirs) { - const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); - if (!(await fs.pathExists(migrationDir))) { - console.log(`No SQL migrations found in ${packageDir}`); - continue; - } - - console.log(`Extracting SQL migrations from ${packageDir}`); - - const migrationFiles = await fs.readdir(migrationDir, { - withFileTypes: true, - }); - - const migrationTargets = migrationFiles - .filter(entry => entry.isDirectory()) - .map(entry => entry.name); - if (migrationFiles.some(entry => entry.isFile())) { - migrationTargets.push('.'); - } - - for (const migrationTarget of migrationTargets) { - await runSingleSqlExtraction(packageDir, migrationTarget, options); - } - } -} - -async function runSingleSqlExtraction( - targetDir: string, - migrationPath: string, - options: SqlExtractionOptions, -) { - const migrationDir = cliPaths.resolveTargetRoot( - targetDir, - 'migrations', - migrationPath, - ); - - console.log(`Extracting SQL from ${migrationDir}`); - const { default: Knex } = await import('knex'); const { default: EmbeddedPostgres } = await import('embedded-postgres'); - console.log(`DEBUG: knex=`, Knex); - console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); + const port = await getPortPromise(); - const port = await getPortPromise({ - /* startPort: 5433, stopPort: 6543 */ - }); - console.log(`DEBUG: port=`, port); - - const pg = new EmbeddedPostgres({ - databaseDir: './data/db', + const basePgOpts = { + host: 'localhost', user: 'postgres', password: 'password', + }; + const pg = new EmbeddedPostgres({ + databaseDir: './data/db', + ...basePgOpts, port, persistent: false, onError(_messageOrError) { @@ -97,18 +60,75 @@ async function runSingleSqlExtraction( // Start the server await pg.start(); - await pg.createDatabase('extractor'); + let dbIndex = 1; - const knex = Knex({ - client: 'pg', - connection: { - host: 'localhost', - port, - user: 'postgres', - password: 'password', - database: 'extractor', - }, - }); + try { + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } + + const { name: pkgName } = await readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); + + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + const database = `extractor-${dbIndex++}`; + await pg.createDatabase(database); + + const knex = Knex({ + client: 'pg', + connection: { + ...basePgOpts, + port, + database, + }, + }); + await runSingleSqlExtraction( + packageDir, + migrationTarget, + pkgName, + knex, + options, + ); + } + } + } finally { + // Stop the server + await pg.stop(); + } +} + +async function runSingleSqlExtraction( + targetDir: string, + migrationTarget: string, + pkgName: string, + knex: Knex, + options: SqlExtractionOptions, +) { + const migrationDir = cliPaths.resolveTargetRoot( + targetDir, + 'migrations', + migrationTarget, + ); + + const reportName = + migrationTarget === '.' ? pkgName : `${pkgName}/${migrationTarget}`; + + console.log(`Generating SQL report for ${reportName}`); const migrationsListResult = await knex.migrate.list({ directory: migrationDir, @@ -120,7 +140,6 @@ async function runSingleSqlExtraction( const schemaInfoBeforeMigration = new Map(); for (const migration of migrations) { - console.log(`DEBUG: UP ${migration}`); const schemaInfo = await getPgSchemaInfo(knex); schemaInfoBeforeMigration.set(migration, schemaInfo); @@ -131,10 +150,9 @@ async function runSingleSqlExtraction( } const schemaInfo = await getPgSchemaInfo(knex); - console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2)); + let failedDownMigration: string | undefined = undefined; for (const migration of migrations.toReversed()) { - console.log(`DEBUG: DOWN ${migration}`); await knex.migrate.down({ directory: migrationDir, name: migration, @@ -146,14 +164,57 @@ async function runSingleSqlExtraction( } const diff = justDiff(before, after); - console.log(`DEBUG: diff=`, diff); if (diff.length !== 0) { - console.log(`Migration ${migration} is not reversible`); - await pg.stop(); - return; + console.log( + `Migration ${migration} is not reversible: ${JSON.stringify( + diff, + null, + 2, + )}`, + ); + failedDownMigration = migration; + break; } } - // Stop the server - await pg.stop(); + const report = generateSqlReport({ + reportName, + failedDownMigration, + schemaInfo, + }); + + const reportPath = cliPaths.resolveTargetRoot( + targetDir, + `report${migrationTarget === '.' ? '' : `-${migrationTarget}`}.sql.md`, + ); + const existingReport = await fs.readFile(reportPath, 'utf8').catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (existingReport !== report) { + if (options.isLocalBuild) { + console.warn(`SQL report changed for ${targetDir}`); + await fs.writeFile(reportPath, report); + } else { + logApiReportInstructions(); + + if (existingReport) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + cliPaths.targetRoot, + reportPath, + )}, expecting the following content:`, + ); + console.log(''); + + console.log(report); + + logApiReportInstructions(); + } + throw new Error(`Report ${reportPath} is out of date`); + } + } } From cdf0a4980a30e98e424a18c4b5590e57f71eca42 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:31:16 +0100 Subject: [PATCH 06/19] repo-tools: nicer error message if embedded-postgres is not installed Signed-off-by: Patrik Oldsberg --- .../commands/api-reports/sql-reports/runSqlExtraction.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 9a78907cfd..ac29f1f027 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -32,7 +32,14 @@ interface SqlExtractionOptions { export async function runSqlExtraction(options: SqlExtractionOptions) { const { default: Knex } = await import('knex'); - const { default: EmbeddedPostgres } = await import('embedded-postgres'); + const { default: EmbeddedPostgres } = await import('embedded-postgres').catch( + error => { + throw new Error( + `Failed to load peer dependency 'embedded-postgres' for generating SQL reports. ` + + `It must be installed as an explicit dependency in your project. Caused by; ${error}`, + ); + }, + ); const port = await getPortPromise(); From 36ed3ed4a151d9400fc65b164e38ce0b30e5cea3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:35:02 +0100 Subject: [PATCH 07/19] repo-tools: stable sql report generation Signed-off-by: Patrik Oldsberg --- .../api-reports/sql-reports/generateSqlReport.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts index 53165590eb..83903e63d0 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts @@ -16,6 +16,10 @@ import { SchemaInfo } from './types'; +function sortedEntries(obj: Record): [string, T][] { + return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)); +} + export function generateSqlReport(options: { reportName: string; failedDownMigration?: string; @@ -39,7 +43,7 @@ export function generateSqlReport(options: { if (Object.keys(schemaInfo.sequences).length > 0) { output.push('## Sequences'); output.push(''); - for (const [sequenceName, sequenceInfo] of Object.entries( + for (const [sequenceName, sequenceInfo] of sortedEntries( schemaInfo.sequences, )) { output.push(`- \`${sequenceName}\` (${sequenceInfo.type})`); @@ -47,12 +51,12 @@ export function generateSqlReport(options: { output.push(''); } - for (const [tableName, tableInfo] of Object.entries(schemaInfo.tables)) { + for (const [tableName, tableInfo] of sortedEntries(schemaInfo.tables)) { output.push(`## Table \`${tableName}\``); output.push(''); output.push(' | Column | Type | Nullable | Max Length | Default |'); output.push(' |--------|------|----------|------------|---------|'); - for (const [columnName, columnInfo] of Object.entries(tableInfo.columns)) { + for (const [columnName, columnInfo] of sortedEntries(tableInfo.columns)) { output.push( ` | \`${columnName}\` | ${columnInfo.type} | ${ columnInfo.nullable @@ -66,7 +70,7 @@ export function generateSqlReport(options: { if (Object.keys(tableInfo.indices).length > 0) { output.push('### Indices'); output.push(''); - for (const [indexName, indexInfo] of Object.entries(tableInfo.indices)) { + for (const [indexName, indexInfo] of sortedEntries(tableInfo.indices)) { const indexType = [ indexInfo.unique && 'unique', indexInfo.primary && 'primary', From a17b865119800e8647d711e5173977472621b713 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:41:10 +0100 Subject: [PATCH 08/19] repo-tools: run prettier on sql reports Signed-off-by: Patrik Oldsberg --- .../sql-reports/runSqlExtraction.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index ac29f1f027..7fa44fe046 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -184,11 +184,13 @@ async function runSingleSqlExtraction( } } - const report = generateSqlReport({ - reportName, - failedDownMigration, - schemaInfo, - }); + const report = prettyReport( + generateSqlReport({ + reportName, + failedDownMigration, + schemaInfo, + }), + ); const reportPath = cliPaths.resolveTargetRoot( targetDir, @@ -225,3 +227,17 @@ async function runSingleSqlExtraction( } } } + +function prettyReport(content: string): string { + try { + const prettier = require('prettier') as typeof import('prettier'); + + const config = prettier.resolveConfig.sync(cliPaths.targetRoot) ?? {}; + return prettier.format(content, { + ...config, + parser: 'markdown', + }); + } catch (e) { + return content; + } +} From d5933e6f345b2bb81fade027fdc3a534c83b3a11 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:46:02 +0100 Subject: [PATCH 09/19] repo-tools: add --sql-reports flag to enable SQL report generation Signed-off-by: Patrik Oldsberg --- package.json | 2 +- packages/repo-tools/cli-report.md | 1 + packages/repo-tools/src/commands/api-reports/api-reports.ts | 2 +- packages/repo-tools/src/commands/index.ts | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 07f3755e63..f4b44fdf52 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:all": "backstage-cli repo build --all", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs --exclude 'plugins/@(adr|adr-backend|adr-common|airbrake|airbrake-backend|allure|analytics-module-ga|analytics-module-ga4|analytics-module-newrelic-browser|apache-airflow|api-docs|api-docs-module-protoc-gen-doc|apollo-explorer|app-visualizer|azure-devops|azure-devops-backend|azure-devops-common|azure-sites|azure-sites-backend|azure-sites-common|badges|badges-backend|bazaar|bazaar-backend|bitbucket-cloud-common|bitrise|catalog-graph|catalog-graphql|catalog-import|catalog-unprocessed-entities|cicd-statistics|cicd-statistics-module-gitlab|circleci|cloudbuild|code-climate|code-coverage|code-coverage-backend|codescene|config-schema|cost-insights|cost-insights-common|dynatrace|entity-feedback|entity-feedback-backend|entity-feedback-common|entity-validation|example-todo-list|example-todo-list-backend|example-todo-list-common|firehydrant|fossa|gcalendar|gcp-projects|git-release-manager|github-actions|github-deployments|github-issues|github-pull-requests-board|gitops-profiles|gocd|graphiql|graphql-backend|graphql-voyager|ilert|jenkins|jenkins-backend|jenkins-common|kafka|kafka-backend|lighthouse|lighthouse-backend|lighthouse-common|linguist|linguist-backend|linguist-common|microsoft-calendar|newrelic|newrelic-dashboard|nomad|nomad-backend|octopus-deploy|opencost|pagerduty|periskop|periskop-backend|playlist|playlist-backend|playlist-common|proxy-backend|puppetdb|rollbar|rollbar-backend|sentry|shortcuts|splunk-on-call|stack-overflow|stack-overflow-backend|stackstorm|tech-radar|tech-radar-2|todo|todo-backend|xcmetrics)'", "build:api-reports": "yarn build:api-reports:only --tsc", - "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags", + "build:api-reports:only": "NODE_OPTIONS=--max-old-space-size=8192 backstage-repo-tools api-reports --sql-reports --allow-warnings 'packages/backend-app-api,packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)' -o ae-undocumented,ae-wrong-input-file-type --validate-release-tags", "build:backend": "yarn workspace example-backend build", "build:knip-reports": "backstage-repo-tools knip-reports", "build:plugins-report": "node ./scripts/build-plugins-report", diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index da483fe016..672c22944a 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -33,6 +33,7 @@ Options: --ci --tsc --docs + --sql-reports --include --exclude -a, --allow-warnings diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 1e10ade111..bec9cfd6c3 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -104,7 +104,7 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => { }); } - if (sqlPackageDirs.length > 0) { + if (sqlPackageDirs.length > 0 && opts.sqlReports) { console.log('# Generating package SQL reports'); await runSqlExtraction({ packageDirs: sqlPackageDirs, diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index e3c4c9e41f..4e984c72de 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -187,6 +187,7 @@ export function registerCommands(program: Command) { .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') + .option('--sql-reports', 'Also generate SQL reports from migration files') .option( '--include ', 'Only include packages matching the provided patterns', From 98ddf0580d98a5ac6a96aa94d3617c90e3f94019 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:55:50 +0100 Subject: [PATCH 10/19] changesets: added changeset for sql reports Signed-off-by: Patrik Oldsberg --- .changeset/cyan-grapes-confess.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cyan-grapes-confess.md diff --git a/.changeset/cyan-grapes-confess.md b/.changeset/cyan-grapes-confess.md new file mode 100644 index 0000000000..c2ccb2054e --- /dev/null +++ b/.changeset/cyan-grapes-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': patch +--- + +The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. In order to generate SQL reports you must also install the desired version of the `embedded-postgres` package as a dependency in your project. From 1abf23efe28a13587f0722dced13b0c114e63f10 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 17:10:18 +0100 Subject: [PATCH 11/19] repo-tools: more codeblocks in SQL reports Signed-off-by: Patrik Oldsberg --- .../sql-reports/generateSqlReport.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts index 83903e63d0..c804c96450 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts @@ -20,6 +20,13 @@ function sortedEntries(obj: Record): [string, T][] { return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)); } +function code(str: unknown): string { + if (str === '-') { + return str; + } + return `\`${str}\``; +} + export function generateSqlReport(options: { reportName: string; failedDownMigration?: string; @@ -46,23 +53,23 @@ export function generateSqlReport(options: { for (const [sequenceName, sequenceInfo] of sortedEntries( schemaInfo.sequences, )) { - output.push(`- \`${sequenceName}\` (${sequenceInfo.type})`); + output.push(`- ${code(sequenceName)} (${sequenceInfo.type})`); } output.push(''); } for (const [tableName, tableInfo] of sortedEntries(schemaInfo.tables)) { - output.push(`## Table \`${tableName}\``); + output.push(`## Table ${code(tableName)}`); output.push(''); output.push(' | Column | Type | Nullable | Max Length | Default |'); output.push(' |--------|------|----------|------------|---------|'); for (const [columnName, columnInfo] of sortedEntries(tableInfo.columns)) { output.push( - ` | \`${columnName}\` | ${columnInfo.type} | ${ + ` | ${code(columnName)} | ${code(columnInfo.type)} | ${ columnInfo.nullable - } | ${columnInfo.maxLength ?? '-'} | ${ - columnInfo.defaultValue ?? '-' - } |`, + } | ${columnInfo.maxLength ?? '-'} | ${code( + columnInfo.defaultValue ?? '-', + )} |`, ); } output.push(''); @@ -78,7 +85,7 @@ export function generateSqlReport(options: { .filter(Boolean) .join(' '); output.push( - `- \`${indexName}\` (\`${indexInfo.columns.join('`, `')}\`)${ + `- ${code(indexName)} (${indexInfo.columns.map(code).join(', ')})${ indexType ? ` ${indexType}` : '' }`, ); From 21985baa7f5637dd5e89f2ccc5f5683c0199cc7f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 17:10:31 +0100 Subject: [PATCH 12/19] generate all SQL reports Signed-off-by: Patrik Oldsberg --- .../config/vocabularies/Backstage/accept.txt | 1 + packages/backend-defaults/report-auth.sql.md | 15 ++ .../backend-defaults/report-scheduler.sql.md | 18 +++ plugins/app-backend/report.sql.md | 17 +++ plugins/auth-backend/report.sql.md | 44 ++++++ .../report.sql.md | 53 +++++++ plugins/catalog-backend/report.sql.md | 137 ++++++++++++++++++ plugins/events-backend/report.sql.md | 38 +++++ plugins/notifications-backend/report.sql.md | 76 ++++++++++ plugins/scaffolder-backend/report.sql.md | 40 +++++ .../search-backend-module-pg/report.sql.md | 19 +++ plugins/user-settings-backend/report.sql.md | 16 ++ 12 files changed, 474 insertions(+) create mode 100644 packages/backend-defaults/report-auth.sql.md create mode 100644 packages/backend-defaults/report-scheduler.sql.md create mode 100644 plugins/app-backend/report.sql.md create mode 100644 plugins/auth-backend/report.sql.md create mode 100644 plugins/catalog-backend-module-incremental-ingestion/report.sql.md create mode 100644 plugins/catalog-backend/report.sql.md create mode 100644 plugins/events-backend/report.sql.md create mode 100644 plugins/notifications-backend/report.sql.md create mode 100644 plugins/scaffolder-backend/report.sql.md create mode 100644 plugins/search-backend-module-pg/report.sql.md create mode 100644 plugins/user-settings-backend/report.sql.md diff --git a/.github/vale/config/vocabularies/Backstage/accept.txt b/.github/vale/config/vocabularies/Backstage/accept.txt index 091b999dd7..7dea58e627 100644 --- a/.github/vale/config/vocabularies/Backstage/accept.txt +++ b/.github/vale/config/vocabularies/Backstage/accept.txt @@ -276,6 +276,7 @@ nohoist nonces noop npm +nullable nunjucks nvarchar nvm diff --git a/packages/backend-defaults/report-auth.sql.md b/packages/backend-defaults/report-auth.sql.md new file mode 100644 index 0000000000..9551c34bb0 --- /dev/null +++ b/packages/backend-defaults/report-auth.sql.md @@ -0,0 +1,15 @@ +## SQL Report file for "@backstage/backend-defaults/auth" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `backstage_backend_public_keys__keys` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | ------------------- | -------- | ---------- | ------- | +| `expires_at` | `character varying` | false | 255 | - | +| `id` | `character varying` | false | 255 | - | +| `key` | `text` | false | - | - | + +### Indices + +- `backstage_backend_public_keys__keys_pkey` (`id`) unique primary diff --git a/packages/backend-defaults/report-scheduler.sql.md b/packages/backend-defaults/report-scheduler.sql.md new file mode 100644 index 0000000000..5542ac48b0 --- /dev/null +++ b/packages/backend-defaults/report-scheduler.sql.md @@ -0,0 +1,18 @@ +## SQL Report file for "@backstage/backend-defaults/scheduler" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `backstage_backend_tasks__tasks` + +| Column | Type | Nullable | Max Length | Default | +| ------------------------ | -------------------------- | -------- | ---------- | ------- | +| `current_run_expires_at` | `timestamp with time zone` | true | - | - | +| `current_run_started_at` | `timestamp with time zone` | true | - | - | +| `current_run_ticket` | `text` | true | - | - | +| `id` | `character varying` | false | 255 | - | +| `next_run_start_at` | `timestamp with time zone` | true | - | - | +| `settings_json` | `text` | false | - | - | + +### Indices + +- `backstage_backend_tasks__tasks_pkey` (`id`) unique primary diff --git a/plugins/app-backend/report.sql.md b/plugins/app-backend/report.sql.md new file mode 100644 index 0000000000..e7bc9780fc --- /dev/null +++ b/plugins/app-backend/report.sql.md @@ -0,0 +1,17 @@ +## SQL Report file for "@backstage/plugin-app-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `static_assets_cache` + +| Column | Type | Nullable | Max Length | Default | +| ------------------ | -------------------------- | -------- | ---------- | ------------------------------ | +| `content` | `bytea` | false | - | - | +| `last_modified_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `namespace` | `character varying` | false | 255 | `'default'::character varying` | +| `path` | `text` | false | - | - | + +### Indices + +- `static_asset_cache_last_modified_at_idx` (`last_modified_at`) +- `static_assets_cache_pkey` (`namespace`, `path`) unique primary diff --git a/plugins/auth-backend/report.sql.md b/plugins/auth-backend/report.sql.md new file mode 100644 index 0000000000..7eef7a33cb --- /dev/null +++ b/plugins/auth-backend/report.sql.md @@ -0,0 +1,44 @@ +## SQL Report file for "@backstage/plugin-auth-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +> [!WARNING] +> Failed to migrate down from '20220321100910_timestamptz_again.js' + +## Table `sessions` + +| Column | Type | Nullable | Max Length | Default | +| --------- | -------------------------- | -------- | ---------- | ------- | +| `expired` | `timestamp with time zone` | false | - | - | +| `sess` | `text` | false | - | - | +| `sid` | `character varying` | false | 255 | - | + +### Indices + +- `sessions_expired_idx` (`expired`) +- `sessions_pkey` (`sid`) unique primary +- `sessions_sid_idx` (`sid`) + +## Table `signing_keys` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `key` | `text` | false | - | - | +| `kid` | `character varying` | false | 255 | - | + +### Indices + +- `signing_keys_pkey` (`kid`) unique primary + +## Table `user_info` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | -------------------------- | -------- | ---------- | ------- | +| `exp` | `timestamp with time zone` | false | - | - | +| `user_entity_ref` | `character varying` | false | 255 | - | +| `user_info` | `text` | false | - | - | + +### Indices + +- `user_info_pkey` (`user_entity_ref`) unique primary diff --git a/plugins/catalog-backend-module-incremental-ingestion/report.sql.md b/plugins/catalog-backend-module-incremental-ingestion/report.sql.md new file mode 100644 index 0000000000..d904d73e0d --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/report.sql.md @@ -0,0 +1,53 @@ +## SQL Report file for "@backstage/plugin-catalog-backend-module-incremental-ingestion" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `ingestion_mark_entities` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | ------------------- | -------- | ---------- | ------- | +| `id` | `uuid` | false | - | - | +| `ingestion_mark_id` | `uuid` | false | - | - | +| `ref` | `character varying` | false | 255 | - | + +### Indices + +- `ingestion_mark_entities_pkey` (`id`) unique primary +- `ingestion_mark_entity_ingestion_mark_id_idx` (`ingestion_mark_id`) + +## Table `ingestion_marks` + +| Column | Type | Nullable | Max Length | Default | +| -------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `cursor` | `json` | true | - | - | +| `id` | `uuid` | false | - | - | +| `ingestion_id` | `uuid` | false | - | - | +| `sequence` | `integer` | false | - | `0` | + +### Indices + +- `ingestion_mark_ingestion_id_idx` (`ingestion_id`) +- `ingestion_marks_pkey` (`id`) unique primary + +## Table `ingestions` + +| Column | Type | Nullable | Max Length | Default | +| ------------------------ | -------------------------- | -------- | ---------- | ------------------- | +| `attempts` | `integer` | false | - | `0` | +| `completion_ticket` | `character varying` | false | 255 | - | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `id` | `uuid` | false | - | - | +| `ingestion_completed_at` | `timestamp with time zone` | true | - | - | +| `last_error` | `character varying` | true | 255 | - | +| `next_action` | `character varying` | false | 255 | - | +| `next_action_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `provider_name` | `character varying` | false | 255 | - | +| `rest_completed_at` | `timestamp with time zone` | true | - | - | +| `status` | `character varying` | false | 255 | - | + +### Indices + +- `ingestion_composite_index` (`provider_name`, `completion_ticket`) unique +- `ingestion_provider_name_idx` (`provider_name`) +- `ingestions_pkey` (`id`) unique primary diff --git a/plugins/catalog-backend/report.sql.md b/plugins/catalog-backend/report.sql.md new file mode 100644 index 0000000000..3402b05063 --- /dev/null +++ b/plugins/catalog-backend/report.sql.md @@ -0,0 +1,137 @@ +## SQL Report file for "@backstage/plugin-catalog-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +> [!WARNING] +> Failed to migrate down from '20241003170511_alter_target_in_locations.js' + +## Sequences + +- `location_update_log_id_seq` (bigint) +- `refresh_state_references_id_seq` (integer) + +## Table `final_entities` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | -------------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | false | 255 | - | +| `entity_ref` | `character varying` | false | 255 | - | +| `final_entity` | `text` | true | - | - | +| `hash` | `character varying` | false | 255 | - | +| `last_updated_at` | `timestamp with time zone` | true | - | - | +| `stitch_ticket` | `text` | false | - | - | + +### Indices + +- `final_entities_entity_ref_uniq` (`entity_ref`) unique +- `final_entities_pkey` (`entity_id`) unique primary + +## Table `location_update_log` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------------------------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `entity_name` | `text` | true | - | - | +| `id` | `bigint` | false | - | `nextval('location_update_log_id_seq'::regclass)` | +| `location_id` | `uuid` | true | - | - | +| `message` | `text` | true | - | - | +| `status` | `text` | false | - | - | + +### Indices + +- `location_update_log_pkey` (`id`) unique primary +- `update_log_location_id_idx` (`location_id`) + +## Table `locations` + +| Column | Type | Nullable | Max Length | Default | +| -------- | ------------------- | -------- | ---------- | ------- | +| `id` | `uuid` | false | - | - | +| `target` | `text` | true | - | - | +| `type` | `character varying` | false | 255 | - | + +### Indices + +- `locations_pkey` (`id`) unique primary + +## Table `refresh_keys` + +| Column | Type | Nullable | Max Length | Default | +| ----------- | ------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | false | 255 | - | +| `key` | `character varying` | false | 255 | - | + +### Indices + +- `refresh_keys_entity_id_idx` (`entity_id`) +- `refresh_keys_key_idx` (`key`) + +## Table `refresh_state` + +| Column | Type | Nullable | Max Length | Default | +| -------------------- | -------------------------- | -------- | ---------- | ------- | +| `cache` | `text` | true | - | - | +| `entity_id` | `character varying` | false | 255 | - | +| `entity_ref` | `character varying` | false | 255 | - | +| `errors` | `text` | false | - | - | +| `last_discovery_at` | `timestamp with time zone` | false | - | - | +| `location_key` | `text` | true | - | - | +| `next_stitch_at` | `timestamp with time zone` | true | - | - | +| `next_stitch_ticket` | `character varying` | true | 255 | - | +| `next_update_at` | `timestamp with time zone` | false | - | - | +| `processed_entity` | `text` | true | - | - | +| `result_hash` | `text` | true | - | - | +| `unprocessed_entity` | `text` | false | - | - | +| `unprocessed_hash` | `text` | true | - | - | + +### Indices + +- `refresh_state_entity_ref_uniq` (`entity_ref`) unique +- `refresh_state_next_stitch_at_idx` (`next_stitch_at`) +- `refresh_state_next_update_at_idx` (`next_update_at`) +- `refresh_state_pkey` (`entity_id`) unique primary + +## Table `refresh_state_references` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | --------- | -------- | ---------- | ------------------------------------------------------ | +| `id` | `integer` | false | - | `nextval('refresh_state_references_id_seq'::regclass)` | +| `source_entity_ref` | `text` | true | - | - | +| `source_key` | `text` | true | - | - | +| `target_entity_ref` | `text` | false | - | - | + +### Indices + +- `refresh_state_references_pkey` (`id`) unique primary +- `refresh_state_references_source_entity_ref_idx` (`source_entity_ref`) +- `refresh_state_references_source_key_idx` (`source_key`) +- `refresh_state_references_target_entity_ref_idx` (`target_entity_ref`) + +## Table `relations` + +| Column | Type | Nullable | Max Length | Default | +| ----------------------- | ------------------- | -------- | ---------- | ------- | +| `originating_entity_id` | `character varying` | false | 255 | - | +| `source_entity_ref` | `character varying` | false | 255 | - | +| `target_entity_ref` | `character varying` | false | 255 | - | +| `type` | `character varying` | false | 255 | - | + +### Indices + +- `relations_source_entity_id_idx` (`originating_entity_id`) +- `relations_source_entity_ref_idx` (`source_entity_ref`) + +## Table `search` + +| Column | Type | Nullable | Max Length | Default | +| ---------------- | ------------------- | -------- | ---------- | ------- | +| `entity_id` | `character varying` | true | 255 | - | +| `key` | `character varying` | false | 255 | - | +| `original_value` | `character varying` | true | 255 | - | +| `value` | `character varying` | true | 255 | - | + +### Indices + +- `search_entity_id_idx` (`entity_id`) +- `search_key_original_value_idx` (`key`, `original_value`) +- `search_key_value_idx` (`key`, `value`) diff --git a/plugins/events-backend/report.sql.md b/plugins/events-backend/report.sql.md new file mode 100644 index 0000000000..117b41afd5 --- /dev/null +++ b/plugins/events-backend/report.sql.md @@ -0,0 +1,38 @@ +## SQL Report file for "@backstage/plugin-events-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Sequences + +- `event_bus_events_id_seq` (bigint) + +## Table `event_bus_events` + +| Column | Type | Nullable | Max Length | Default | +| ---------------------- | -------------------------- | -------- | ---------- | ---------------------------------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | false | - | - | +| `data_json` | `text` | false | - | - | +| `id` | `bigint` | false | - | `nextval('event_bus_events_id_seq'::regclass)` | +| `notified_subscribers` | `ARRAY` | true | - | - | +| `topic` | `text` | false | - | - | + +### Indices + +- `event_bus_events_pkey` (`id`) unique primary +- `event_bus_events_topic_idx` (`topic`) + +## Table `event_bus_subscriptions` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | false | - | - | +| `id` | `character varying` | false | 255 | - | +| `read_until` | `bigint` | false | - | - | +| `topics` | `ARRAY` | true | - | - | +| `updated_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | + +### Indices + +- `event_bus_subscriptions_pkey` (`id`) unique primary diff --git a/plugins/notifications-backend/report.sql.md b/plugins/notifications-backend/report.sql.md new file mode 100644 index 0000000000..6b9066e77c --- /dev/null +++ b/plugins/notifications-backend/report.sql.md @@ -0,0 +1,76 @@ +## SQL Report file for "@backstage/plugin-notifications-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `broadcast` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `description` | `text` | true | - | - | +| `icon` | `character varying` | true | 255 | - | +| `id` | `uuid` | false | - | - | +| `link` | `text` | true | - | - | +| `origin` | `character varying` | false | 255 | - | +| `scope` | `character varying` | true | 255 | - | +| `severity` | `character varying` | false | 8 | - | +| `title` | `character varying` | false | 255 | - | +| `topic` | `character varying` | true | 255 | - | +| `updated` | `timestamp with time zone` | true | - | - | + +### Indices + +- `broadcast_cope_origin_idx` (`scope`, `origin`) +- `broadcast_pkey` (`id`) unique primary + +## Table `broadcast_user_status` + +| Column | Type | Nullable | Max Length | Default | +| -------------- | -------------------------- | -------- | ---------- | ------- | +| `broadcast_id` | `uuid` | false | - | - | +| `read` | `timestamp with time zone` | true | - | - | +| `saved` | `timestamp with time zone` | true | - | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `broadcast_user_idx` (`broadcast_id`, `user`) unique + +## Table `notification` + +| Column | Type | Nullable | Max Length | Default | +| ------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `description` | `text` | true | - | - | +| `icon` | `character varying` | true | 255 | - | +| `id` | `uuid` | false | - | - | +| `link` | `text` | true | - | - | +| `origin` | `character varying` | false | 255 | - | +| `read` | `timestamp with time zone` | true | - | - | +| `saved` | `timestamp with time zone` | true | - | - | +| `scope` | `character varying` | true | 255 | - | +| `severity` | `character varying` | false | 8 | - | +| `title` | `character varying` | false | 255 | - | +| `topic` | `character varying` | true | 255 | - | +| `updated` | `timestamp with time zone` | true | - | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `notification_pkey` (`id`) unique primary +- `notification_scope_origin_idx` (`scope`, `origin`) +- `notification_user_idx` (`user`) + +## Table `user_settings` + +| Column | Type | Nullable | Max Length | Default | +| --------- | ------------------- | -------- | ---------- | ------- | +| `channel` | `character varying` | false | 255 | - | +| `enabled` | `boolean` | false | - | `true` | +| `origin` | `character varying` | false | 255 | - | +| `user` | `character varying` | false | 255 | - | + +### Indices + +- `user_settings_unique_idx` (`user`, `channel`, `origin`) unique +- `user_settings_user_idx` (`user`) diff --git a/plugins/scaffolder-backend/report.sql.md b/plugins/scaffolder-backend/report.sql.md new file mode 100644 index 0000000000..34c57266aa --- /dev/null +++ b/plugins/scaffolder-backend/report.sql.md @@ -0,0 +1,40 @@ +## SQL Report file for "@backstage/plugin-scaffolder-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Sequences + +- `task_events_id_seq` (bigint) + +## Table `task_events` + +| Column | Type | Nullable | Max Length | Default | +| ------------ | -------------------------- | -------- | ---------- | ----------------------------------------- | +| `body` | `text` | false | - | - | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `event_type` | `text` | false | - | - | +| `id` | `bigint` | false | - | `nextval('task_events_id_seq'::regclass)` | +| `task_id` | `uuid` | false | - | - | + +### Indices + +- `task_events_pkey` (`id`) unique primary +- `task_events_task_id_idx` (`task_id`) + +## Table `tasks` + +| Column | Type | Nullable | Max Length | Default | +| ------------------- | -------------------------- | -------- | ---------- | ------------------- | +| `created_at` | `timestamp with time zone` | false | - | `CURRENT_TIMESTAMP` | +| `created_by` | `text` | true | - | - | +| `id` | `uuid` | false | - | - | +| `last_heartbeat_at` | `timestamp with time zone` | true | - | - | +| `secrets` | `text` | true | - | - | +| `spec` | `text` | false | - | - | +| `state` | `text` | true | - | - | +| `status` | `text` | false | - | - | +| `workspace` | `bytea` | true | - | - | + +### Indices + +- `tasks_pkey` (`id`) unique primary diff --git a/plugins/search-backend-module-pg/report.sql.md b/plugins/search-backend-module-pg/report.sql.md new file mode 100644 index 0000000000..ebb5bad3e4 --- /dev/null +++ b/plugins/search-backend-module-pg/report.sql.md @@ -0,0 +1,19 @@ +## SQL Report file for "@backstage/plugin-search-backend-module-pg" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `documents` + +| Column | Type | Nullable | Max Length | Default | +| ---------- | ---------- | -------- | ---------- | ------- | +| `body` | `tsvector` | false | - | - | +| `document` | `jsonb` | false | - | - | +| `hash` | `bytea` | false | - | - | +| `type` | `text` | false | - | - | + +### Indices + +- `documents_body_index` (`body`) +- `documents_document_index` (`document`) +- `documents_pkey` (`hash`) unique primary +- `documents_type_index` (`type`) diff --git a/plugins/user-settings-backend/report.sql.md b/plugins/user-settings-backend/report.sql.md new file mode 100644 index 0000000000..bb312cf015 --- /dev/null +++ b/plugins/user-settings-backend/report.sql.md @@ -0,0 +1,16 @@ +## SQL Report file for "@backstage/plugin-user-settings-backend" + +> Do not edit this file. It is a report generated by `yarn build:api-reports` + +## Table `user_settings` + +| Column | Type | Nullable | Max Length | Default | +| ----------------- | ------------------- | -------- | ---------- | ------- | +| `bucket` | `character varying` | false | 255 | - | +| `key` | `character varying` | false | 255 | - | +| `user_entity_ref` | `character varying` | false | 255 | - | +| `value` | `text` | false | - | - | + +### Indices + +- `user_settings_pkey` (`user_entity_ref`, `bucket`, `key`) unique primary From d4adf46996745b1a395c9bcefad85a6973f1a129 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 19:01:23 +0100 Subject: [PATCH 13/19] repo-tools: fix api-reports test Signed-off-by: Patrik Oldsberg --- packages/repo-tools/src/commands/api-reports/api-reports.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts index 94ae42d715..02de908b8e 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -38,6 +38,7 @@ jest.mock('./api-extractor', () => ({ return { tsPackageDirs: p, cliPackageDirs: p, + sqlPackageDirs: [], }; }), runApiExtraction: jest.fn(), From 2e0a239c1ededbddb5bcc6a9fd0b3fbbd40fc712 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 19:01:57 +0100 Subject: [PATCH 14/19] repo-tools: fix module type declaration for embedded-postgres Signed-off-by: Patrik Oldsberg --- packages/repo-tools/src/types.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/repo-tools/src/types.d.ts b/packages/repo-tools/src/types.d.ts index 276b6ec8f9..2e30114f10 100644 --- a/packages/repo-tools/src/types.d.ts +++ b/packages/repo-tools/src/types.d.ts @@ -16,5 +16,5 @@ // It's missing a types entry point, but has types in dist declare module 'embedded-postgres' { - export { default } from 'embedded-postgres/dist/index.d.ts'; + export { default } from 'embedded-postgres/dist/index'; } From 3051f26bbd7336dfcd0889c8528bd843060adbfd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 19:02:49 +0100 Subject: [PATCH 15/19] repo-tools: removed dead code Signed-off-by: Patrik Oldsberg --- .../commands/api-reports/sql-reports/runSqlExtraction.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 7fa44fe046..8c9b914662 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -53,12 +53,8 @@ export async function runSqlExtraction(options: SqlExtractionOptions) { ...basePgOpts, port, persistent: false, - onError(_messageOrError) { - // console.error('EmbeddedPostgres error:', messageOrError); - }, - onLog(_message) { - // console.log('EmbeddedPostgres log:', message); - }, + onError(_messageOrError) {}, + onLog(_message) {}, }); // Create the cluster config files From ccf5fa8cc55a0397ad575a5d39127b1c51e610c3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 29 Dec 2024 02:00:41 +0100 Subject: [PATCH 16/19] repo-tools: filter out SQL reports when checking for extra reports Signed-off-by: Patrik Oldsberg --- packages/repo-tools/src/commands/api-reports/api-extractor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 4db8077501..0bac1cbeb1 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -398,6 +398,7 @@ export async function runApiExtraction({ filename => // https://regex101.com/r/QDZIV0/2 filename !== 'knip-report.md' && + !filename.endsWith('.sql.md') && // this has to temporarily match all old api report formats filename.match(/^.*?(api-)?report(-[^.-]+)?(.*?)\.md$/), ), From da26a15cec7e7073a2749f893b933e0f72497b23 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 31 Dec 2024 15:29:14 +0100 Subject: [PATCH 17/19] chore: pg-lite experiment Signed-off-by: blam --- packages/repo-tools/package.json | 2 + .../api-reports/sql-reports/pglite.ts | 138 ++++++++++++++++++ .../sql-reports/runSqlExtraction.ts | 46 ++---- 3 files changed, 149 insertions(+), 37 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index b5f1bbc596..eeea5e62ee 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -49,6 +49,7 @@ "@backstage/cli-node": "workspace:^", "@backstage/config-loader": "workspace:^", "@backstage/errors": "workspace:^", + "@electric-sql/pglite": "^0.2.15", "@manypkg/get-packages": "^1.1.3", "@microsoft/api-documenter": "^7.25.7", "@microsoft/api-extractor": "^7.47.2", @@ -72,6 +73,7 @@ "js-yaml": "^4.1.0", "just-diff": "^6.0.2", "knex": "^3.0.0", + "knex-pglite": "^0.11.0", "lodash": "^4.17.21", "minimatch": "^9.0.0", "p-limit": "^3.0.2", diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts new file mode 100644 index 0000000000..96b53f893b --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PGlite } from '@electric-sql/pglite'; +import { Client, Knex } from 'knex'; + +const Client_PG: any = require('knex/lib/dialects/postgres/index.js'); + +type KnexPGliteConfig = Knex.Config & { connection: { pglite?: PGlite } }; + +export class PgLiteClient extends Client_PG { + private pglite: PGlite | undefined; + + constructor(config: KnexPGliteConfig) { + super({ + ...config, + client: 'pg', + // Enforce a single connection: + pool: { min: 1, max: 1 }, + } satisfies Knex.Config); + if (config.pool) { + throw new Error( + 'PGlite is single user/connection. Pool cannot be configured.', + ); + } + this.pglite = + config.connection?.pglite ?? + new PGlite( + config.connection?.['filename'] ?? + config.connection?.['connectionString'], + ); + } + + _driver() {} + + async _acquireOnlyConnection() { + const connection = this.pglite; + await connection.waitReady; + return connection; + } + + async destroyRawConnection(connection: PGlite) { + // There is only one connection, if this one goes shut down the database + await connection.close(); + } + + async setSchemaSearchPath( + connection: PGlite, + searchPath: string, + ): Promise { + let path = searchPath || this.searchPath; + + if (!path) { + return true; + } + + if (!Array.isArray(path) && typeof path !== 'string') { + throw new TypeError( + `knex: Expected searchPath to be Array/String, got: ${typeof path}`, + ); + } + + if (typeof path === 'string') { + if (path.includes(',')) { + const parts = path.split(','); + const arraySyntax = `[${parts + .map(searchPath => `'${searchPath}'`) + .join(', ')}]`; + this.logger.warn?.( + `Detected comma in searchPath "${path}".` + + `If you are trying to specify multiple schemas, use Array syntax: ${arraySyntax}`, + ); + } + path = [path]; + } + + path = path.map((schemaName: string) => `"${schemaName}"`).join(','); + + await connection.query(`set search_path to ${path}`); + return true; + } + + async checkVersion(connection: PGlite) { + const resp = await connection.query('select version();'); + return this._parseVersion((resp.rows[0] as any).version); + } + + async _query(connection: PGlite, obj: any) { + if (!obj.sql) throw new Error('The query is empty'); + + const response = await connection.query(obj.sql, obj.bindings, obj.options); + obj.response = response; + return obj; + } + + processResponse(obj: any, runner: any) { + const response = { + ...obj.response, + rowCount: obj.response.affectedRows, + command: (obj.method as string)?.toUpperCase() ?? '', + }; + return super.processResponse({ ...obj, response }, runner); + } + + _stream(connection: PGlite, obj: any, stream: any) { + return new Promise((resolver, rejecter) => { + stream.on('error', rejecter); + stream.on('end', resolver); + + return this._query(connection, obj) + .then(obj => obj.response.rows) + .then(rows => rows.forEach((row: any) => stream.write(row))) + .catch(err => { + stream.emit('error', err); + }) + .then(() => { + stream.end(); + }); + }); + } +} + +Object.assign(PgLiteClient.prototype, { + // The "dialect", for reference . + driverName: 'postgres', +}); diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 8c9b914662..aa3f4ce066 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -24,6 +24,7 @@ import { getPgSchemaInfo } from './getPgSchemaInfo'; import { generateSqlReport } from './generateSqlReport'; import type { Knex } from 'knex'; import { logApiReportInstructions } from '../api-extractor'; +import { PgLiteClient } from './pglite'; interface SqlExtractionOptions { packageDirs: string[]; @@ -32,36 +33,6 @@ interface SqlExtractionOptions { export async function runSqlExtraction(options: SqlExtractionOptions) { const { default: Knex } = await import('knex'); - const { default: EmbeddedPostgres } = await import('embedded-postgres').catch( - error => { - throw new Error( - `Failed to load peer dependency 'embedded-postgres' for generating SQL reports. ` + - `It must be installed as an explicit dependency in your project. Caused by; ${error}`, - ); - }, - ); - - const port = await getPortPromise(); - - const basePgOpts = { - host: 'localhost', - user: 'postgres', - password: 'password', - }; - const pg = new EmbeddedPostgres({ - databaseDir: './data/db', - ...basePgOpts, - port, - persistent: false, - onError(_messageOrError) {}, - onLog(_message) {}, - }); - - // Create the cluster config files - await pg.initialise(); - - // Start the server - await pg.start(); let dbIndex = 1; @@ -90,16 +61,17 @@ export async function runSqlExtraction(options: SqlExtractionOptions) { for (const migrationTarget of migrationTargets) { const database = `extractor-${dbIndex++}`; - await pg.createDatabase(database); const knex = Knex({ - client: 'pg', + client: PgLiteClient, + dialect: 'postgres', connection: { - ...basePgOpts, - port, database, }, }); + + await knex.raw(`CREATE DATABASE "${database}"`); + await runSingleSqlExtraction( packageDir, migrationTarget, @@ -109,9 +81,9 @@ export async function runSqlExtraction(options: SqlExtractionOptions) { ); } } - } finally { - // Stop the server - await pg.stop(); + } catch (error) { + console.error(error); + process.exit(1); } } From c6ff91a5d8814f3975aaf4ea82cc651d122508fb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 10 Jan 2025 14:35:54 +0100 Subject: [PATCH 18/19] repo-tools: fully move to pglite Signed-off-by: Patrik Oldsberg --- .changeset/cyan-grapes-confess.md | 2 +- packages/repo-tools/package.json | 7 +- .../api-reports/sql-reports/pglite.ts | 138 ------------------ .../sql-reports/runSqlExtraction.ts | 89 +++++------ packages/repo-tools/src/types.d.ts | 20 --- yarn.lock | 112 +++----------- 6 files changed, 65 insertions(+), 303 deletions(-) delete mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts delete mode 100644 packages/repo-tools/src/types.d.ts diff --git a/.changeset/cyan-grapes-confess.md b/.changeset/cyan-grapes-confess.md index c2ccb2054e..a113f91fd9 100644 --- a/.changeset/cyan-grapes-confess.md +++ b/.changeset/cyan-grapes-confess.md @@ -2,4 +2,4 @@ '@backstage/repo-tools': patch --- -The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. In order to generate SQL reports you must also install the desired version of the `embedded-postgres` package as a dependency in your project. +The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index eeea5e62ee..f5c6a0c060 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -88,22 +88,17 @@ "@backstage/types": "workspace:^", "@types/is-glob": "^4.0.2", "@types/node": "^20.16.0", - "@types/prettier": "^2.0.0", - "embedded-postgres": "17.2.0-beta.15" + "@types/prettier": "^2.0.0" }, "peerDependencies": { "@microsoft/api-extractor-model": "*", "@microsoft/tsdoc": "*", "@microsoft/tsdoc-config": "*", "@useoptic/optic": "^1.0.0", - "embedded-postgres": "17.2.0-beta.15", "prettier": "^2.8.1", "typescript": "> 3.0.0" }, "peerDependenciesMeta": { - "embedded-postgres": { - "optional": true - }, "prettier": { "optional": true } diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts deleted file mode 100644 index 96b53f893b..0000000000 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2024 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { PGlite } from '@electric-sql/pglite'; -import { Client, Knex } from 'knex'; - -const Client_PG: any = require('knex/lib/dialects/postgres/index.js'); - -type KnexPGliteConfig = Knex.Config & { connection: { pglite?: PGlite } }; - -export class PgLiteClient extends Client_PG { - private pglite: PGlite | undefined; - - constructor(config: KnexPGliteConfig) { - super({ - ...config, - client: 'pg', - // Enforce a single connection: - pool: { min: 1, max: 1 }, - } satisfies Knex.Config); - if (config.pool) { - throw new Error( - 'PGlite is single user/connection. Pool cannot be configured.', - ); - } - this.pglite = - config.connection?.pglite ?? - new PGlite( - config.connection?.['filename'] ?? - config.connection?.['connectionString'], - ); - } - - _driver() {} - - async _acquireOnlyConnection() { - const connection = this.pglite; - await connection.waitReady; - return connection; - } - - async destroyRawConnection(connection: PGlite) { - // There is only one connection, if this one goes shut down the database - await connection.close(); - } - - async setSchemaSearchPath( - connection: PGlite, - searchPath: string, - ): Promise { - let path = searchPath || this.searchPath; - - if (!path) { - return true; - } - - if (!Array.isArray(path) && typeof path !== 'string') { - throw new TypeError( - `knex: Expected searchPath to be Array/String, got: ${typeof path}`, - ); - } - - if (typeof path === 'string') { - if (path.includes(',')) { - const parts = path.split(','); - const arraySyntax = `[${parts - .map(searchPath => `'${searchPath}'`) - .join(', ')}]`; - this.logger.warn?.( - `Detected comma in searchPath "${path}".` + - `If you are trying to specify multiple schemas, use Array syntax: ${arraySyntax}`, - ); - } - path = [path]; - } - - path = path.map((schemaName: string) => `"${schemaName}"`).join(','); - - await connection.query(`set search_path to ${path}`); - return true; - } - - async checkVersion(connection: PGlite) { - const resp = await connection.query('select version();'); - return this._parseVersion((resp.rows[0] as any).version); - } - - async _query(connection: PGlite, obj: any) { - if (!obj.sql) throw new Error('The query is empty'); - - const response = await connection.query(obj.sql, obj.bindings, obj.options); - obj.response = response; - return obj; - } - - processResponse(obj: any, runner: any) { - const response = { - ...obj.response, - rowCount: obj.response.affectedRows, - command: (obj.method as string)?.toUpperCase() ?? '', - }; - return super.processResponse({ ...obj, response }, runner); - } - - _stream(connection: PGlite, obj: any, stream: any) { - return new Promise((resolver, rejecter) => { - stream.on('error', rejecter); - stream.on('end', resolver); - - return this._query(connection, obj) - .then(obj => obj.response.rows) - .then(rows => rows.forEach((row: any) => stream.write(row))) - .catch(err => { - stream.emit('error', err); - }) - .then(() => { - stream.end(); - }); - }); - } -} - -Object.assign(PgLiteClient.prototype, { - // The "dialect", for reference . - driverName: 'postgres', -}); diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index aa3f4ce066..f3af1c4fc4 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -17,14 +17,12 @@ import fs, { readJson } from 'fs-extra'; import { relative as relativePath } from 'path'; import { paths as cliPaths } from '../../../lib/paths'; -import { getPortPromise } from 'portfinder'; import { diff as justDiff } from 'just-diff'; import { SchemaInfo } from './types'; import { getPgSchemaInfo } from './getPgSchemaInfo'; import { generateSqlReport } from './generateSqlReport'; import type { Knex } from 'knex'; import { logApiReportInstructions } from '../api-extractor'; -import { PgLiteClient } from './pglite'; interface SqlExtractionOptions { packageDirs: string[]; @@ -33,57 +31,60 @@ interface SqlExtractionOptions { export async function runSqlExtraction(options: SqlExtractionOptions) { const { default: Knex } = await import('knex'); + const { default: ClientPgLite } = await import('knex-pglite'); + + // Since we're passing this as the client we need to replace the `config.client` with `pg` afterwards + class WrappedClientPgLite extends ClientPgLite { + constructor(config: any) { + super({ ...config, client: 'pg' }); + } + } let dbIndex = 1; - try { - for (const packageDir of options.packageDirs) { - const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); - if (!(await fs.pathExists(migrationDir))) { - console.log(`No SQL migrations found in ${packageDir}`); - continue; - } + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } - const { name: pkgName } = await readJson( - cliPaths.resolveTargetRoot(packageDir, 'package.json'), - ); + const { name: pkgName } = await readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); - const migrationFiles = await fs.readdir(migrationDir, { - withFileTypes: true, + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + const database = `extractor-${dbIndex++}`; + + const knex = Knex({ + client: WrappedClientPgLite, + dialect: 'postgres', + connection: { + database, + }, }); - const migrationTargets = migrationFiles - .filter(entry => entry.isDirectory()) - .map(entry => entry.name); - if (migrationFiles.some(entry => entry.isFile())) { - migrationTargets.push('.'); - } + await knex.raw(`CREATE DATABASE "${database}"`); - for (const migrationTarget of migrationTargets) { - const database = `extractor-${dbIndex++}`; - - const knex = Knex({ - client: PgLiteClient, - dialect: 'postgres', - connection: { - database, - }, - }); - - await knex.raw(`CREATE DATABASE "${database}"`); - - await runSingleSqlExtraction( - packageDir, - migrationTarget, - pkgName, - knex, - options, - ); - } + await runSingleSqlExtraction( + packageDir, + migrationTarget, + pkgName, + knex, + options, + ); } - } catch (error) { - console.error(error); - process.exit(1); } } diff --git a/packages/repo-tools/src/types.d.ts b/packages/repo-tools/src/types.d.ts deleted file mode 100644 index 2e30114f10..0000000000 --- a/packages/repo-tools/src/types.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2024 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// It's missing a types entry point, but has types in dist -declare module 'embedded-postgres' { - export { default } from 'embedded-postgres/dist/index'; -} diff --git a/yarn.lock b/yarn.lock index 0c67429722..1b70dfda3b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8293,6 +8293,7 @@ __metadata: "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" + "@electric-sql/pglite": ^0.2.15 "@manypkg/get-packages": ^1.1.3 "@microsoft/api-documenter": ^7.25.7 "@microsoft/api-extractor": ^7.47.2 @@ -8313,13 +8314,13 @@ __metadata: codeowners-utils: ^1.0.2 command-exists: ^1.2.9 commander: ^12.0.0 - embedded-postgres: 17.2.0-beta.15 fs-extra: ^11.2.0 glob: ^8.0.3 is-glob: ^4.0.3 js-yaml: ^4.1.0 just-diff: ^6.0.2 knex: ^3.0.0 + knex-pglite: ^0.11.0 lodash: ^4.17.21 minimatch: ^9.0.0 p-limit: ^3.0.2 @@ -8332,12 +8333,9 @@ __metadata: "@microsoft/tsdoc": "*" "@microsoft/tsdoc-config": "*" "@useoptic/optic": ^1.0.0 - embedded-postgres: 17.2.0-beta.15 prettier: ^2.8.1 typescript: "> 3.0.0" peerDependenciesMeta: - embedded-postgres: - optional: true prettier: optional: true bin: @@ -8964,59 +8962,10 @@ __metadata: languageName: node linkType: hard -"@embedded-postgres/darwin-arm64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/darwin-arm64@npm:17.2.0-beta.15" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - -"@embedded-postgres/darwin-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/darwin-x64@npm:17.2.0-beta.15" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-arm64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-arm64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=arm64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-arm@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-arm@npm:17.2.0-beta.15" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - -"@embedded-postgres/linux-ia32@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-ia32@npm:17.2.0-beta.15" - conditions: os=linux & cpu=ia32 - languageName: node - linkType: hard - -"@embedded-postgres/linux-ppc64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-ppc64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=ppc64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-x64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=x64 - languageName: node - linkType: hard - -"@embedded-postgres/windows-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/windows-x64@npm:17.2.0-beta.15" - conditions: os=win32 & cpu=x64 +"@electric-sql/pglite@npm:^0.2.14, @electric-sql/pglite@npm:^0.2.15": + version: 0.2.15 + resolution: "@electric-sql/pglite@npm:0.2.15" + checksum: 714136449bdbef92ffc8475ba2237418bbdcef3bdbeb0eb43ab2e9f5fc4c2d9c67c41cbcd1e5ad7ba758d22f1bfb996a3f372745509845e55d9f69349f396aca languageName: node linkType: hard @@ -26898,41 +26847,6 @@ __metadata: languageName: node linkType: hard -"embedded-postgres@npm:17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "embedded-postgres@npm:17.2.0-beta.15" - dependencies: - "@embedded-postgres/darwin-arm64": ^17.2.0-beta.15 - "@embedded-postgres/darwin-x64": ^17.2.0-beta.15 - "@embedded-postgres/linux-arm": ^17.2.0-beta.15 - "@embedded-postgres/linux-arm64": ^17.2.0-beta.15 - "@embedded-postgres/linux-ia32": ^17.2.0-beta.15 - "@embedded-postgres/linux-ppc64": ^17.2.0-beta.15 - "@embedded-postgres/linux-x64": ^17.2.0-beta.15 - "@embedded-postgres/windows-x64": ^17.2.0-beta.15 - async-exit-hook: ^2.0.1 - pg: ^8.7.3 - dependenciesMeta: - "@embedded-postgres/darwin-arm64": - optional: true - "@embedded-postgres/darwin-x64": - optional: true - "@embedded-postgres/linux-arm": - optional: true - "@embedded-postgres/linux-arm64": - optional: true - "@embedded-postgres/linux-ia32": - optional: true - "@embedded-postgres/linux-ppc64": - optional: true - "@embedded-postgres/linux-x64": - optional: true - "@embedded-postgres/windows-x64": - optional: true - checksum: 53b261693dcc83cea6cc2589794dbc69bcf4cae508f2021d4686d7fb08e686e93db97392ebd28dfd2bb0126060b54fa609487d9929a4fee19da6d66dc568863f - languageName: node - linkType: hard - "emittery@npm:^0.13.1": version: 0.13.1 resolution: "emittery@npm:0.13.1" @@ -33807,7 +33721,17 @@ __metadata: languageName: node linkType: hard -"knex@npm:3, knex@npm:^3.0.0": +"knex-pglite@npm:^0.11.0": + version: 0.11.0 + resolution: "knex-pglite@npm:0.11.0" + dependencies: + "@electric-sql/pglite": ^0.2.14 + knex: 3.1.0 + checksum: 680de87b9c567bfc3009c81e85584e4e634ede8b98f14acd4d4463e1eae084cdfc0eed05979db742ee2af1bd6f857bb23ca28c4175abdf27b3d6e3e2dfcba4df + languageName: node + linkType: hard + +"knex@npm:3, knex@npm:3.1.0, knex@npm:^3.0.0": version: 3.1.0 resolution: "knex@npm:3.1.0" dependencies: @@ -38487,7 +38411,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": +"pg@npm:^8.11.3, pg@npm:^8.9.0": version: 8.13.1 resolution: "pg@npm:8.13.1" dependencies: From d37913877db525216a0050fa70d748e1ee78c545 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 14 Jan 2025 15:40:46 +0100 Subject: [PATCH 19/19] Update packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../src/commands/api-reports/sql-reports/runSqlExtraction.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index f3af1c4fc4..001c4c6c45 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -75,7 +75,7 @@ export async function runSqlExtraction(options: SqlExtractionOptions) { }, }); - await knex.raw(`CREATE DATABASE "${database}"`); + await knex.raw('CREATE DATABASE ??', [database]); await runSingleSqlExtraction( packageDir,