diff --git a/.changeset/cyan-grapes-confess.md b/.changeset/cyan-grapes-confess.md index c2ccb2054e..a113f91fd9 100644 --- a/.changeset/cyan-grapes-confess.md +++ b/.changeset/cyan-grapes-confess.md @@ -2,4 +2,4 @@ '@backstage/repo-tools': patch --- -The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. In order to generate SQL reports you must also install the desired version of the `embedded-postgres` package as a dependency in your project. +The `api-reports` command is now also able to generate SQL reports, enabled by the `--sql-reports` flag. diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index eeea5e62ee..f5c6a0c060 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -88,22 +88,17 @@ "@backstage/types": "workspace:^", "@types/is-glob": "^4.0.2", "@types/node": "^20.16.0", - "@types/prettier": "^2.0.0", - "embedded-postgres": "17.2.0-beta.15" + "@types/prettier": "^2.0.0" }, "peerDependencies": { "@microsoft/api-extractor-model": "*", "@microsoft/tsdoc": "*", "@microsoft/tsdoc-config": "*", "@useoptic/optic": "^1.0.0", - "embedded-postgres": "17.2.0-beta.15", "prettier": "^2.8.1", "typescript": "> 3.0.0" }, "peerDependenciesMeta": { - "embedded-postgres": { - "optional": true - }, "prettier": { "optional": true } diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts deleted file mode 100644 index 96b53f893b..0000000000 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2024 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { PGlite } from '@electric-sql/pglite'; -import { Client, Knex } from 'knex'; - -const Client_PG: any = require('knex/lib/dialects/postgres/index.js'); - -type KnexPGliteConfig = Knex.Config & { connection: { pglite?: PGlite } }; - -export class PgLiteClient extends Client_PG { - private pglite: PGlite | undefined; - - constructor(config: KnexPGliteConfig) { - super({ - ...config, - client: 'pg', - // Enforce a single connection: - pool: { min: 1, max: 1 }, - } satisfies Knex.Config); - if (config.pool) { - throw new Error( - 'PGlite is single user/connection. Pool cannot be configured.', - ); - } - this.pglite = - config.connection?.pglite ?? - new PGlite( - config.connection?.['filename'] ?? - config.connection?.['connectionString'], - ); - } - - _driver() {} - - async _acquireOnlyConnection() { - const connection = this.pglite; - await connection.waitReady; - return connection; - } - - async destroyRawConnection(connection: PGlite) { - // There is only one connection, if this one goes shut down the database - await connection.close(); - } - - async setSchemaSearchPath( - connection: PGlite, - searchPath: string, - ): Promise { - let path = searchPath || this.searchPath; - - if (!path) { - return true; - } - - if (!Array.isArray(path) && typeof path !== 'string') { - throw new TypeError( - `knex: Expected searchPath to be Array/String, got: ${typeof path}`, - ); - } - - if (typeof path === 'string') { - if (path.includes(',')) { - const parts = path.split(','); - const arraySyntax = `[${parts - .map(searchPath => `'${searchPath}'`) - .join(', ')}]`; - this.logger.warn?.( - `Detected comma in searchPath "${path}".` + - `If you are trying to specify multiple schemas, use Array syntax: ${arraySyntax}`, - ); - } - path = [path]; - } - - path = path.map((schemaName: string) => `"${schemaName}"`).join(','); - - await connection.query(`set search_path to ${path}`); - return true; - } - - async checkVersion(connection: PGlite) { - const resp = await connection.query('select version();'); - return this._parseVersion((resp.rows[0] as any).version); - } - - async _query(connection: PGlite, obj: any) { - if (!obj.sql) throw new Error('The query is empty'); - - const response = await connection.query(obj.sql, obj.bindings, obj.options); - obj.response = response; - return obj; - } - - processResponse(obj: any, runner: any) { - const response = { - ...obj.response, - rowCount: obj.response.affectedRows, - command: (obj.method as string)?.toUpperCase() ?? '', - }; - return super.processResponse({ ...obj, response }, runner); - } - - _stream(connection: PGlite, obj: any, stream: any) { - return new Promise((resolver, rejecter) => { - stream.on('error', rejecter); - stream.on('end', resolver); - - return this._query(connection, obj) - .then(obj => obj.response.rows) - .then(rows => rows.forEach((row: any) => stream.write(row))) - .catch(err => { - stream.emit('error', err); - }) - .then(() => { - stream.end(); - }); - }); - } -} - -Object.assign(PgLiteClient.prototype, { - // The "dialect", for reference . - driverName: 'postgres', -}); diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index aa3f4ce066..f3af1c4fc4 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -17,14 +17,12 @@ import fs, { readJson } from 'fs-extra'; import { relative as relativePath } from 'path'; import { paths as cliPaths } from '../../../lib/paths'; -import { getPortPromise } from 'portfinder'; import { diff as justDiff } from 'just-diff'; import { SchemaInfo } from './types'; import { getPgSchemaInfo } from './getPgSchemaInfo'; import { generateSqlReport } from './generateSqlReport'; import type { Knex } from 'knex'; import { logApiReportInstructions } from '../api-extractor'; -import { PgLiteClient } from './pglite'; interface SqlExtractionOptions { packageDirs: string[]; @@ -33,57 +31,60 @@ interface SqlExtractionOptions { export async function runSqlExtraction(options: SqlExtractionOptions) { const { default: Knex } = await import('knex'); + const { default: ClientPgLite } = await import('knex-pglite'); + + // Since we're passing this as the client we need to replace the `config.client` with `pg` afterwards + class WrappedClientPgLite extends ClientPgLite { + constructor(config: any) { + super({ ...config, client: 'pg' }); + } + } let dbIndex = 1; - try { - for (const packageDir of options.packageDirs) { - const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); - if (!(await fs.pathExists(migrationDir))) { - console.log(`No SQL migrations found in ${packageDir}`); - continue; - } + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } - const { name: pkgName } = await readJson( - cliPaths.resolveTargetRoot(packageDir, 'package.json'), - ); + const { name: pkgName } = await readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); - const migrationFiles = await fs.readdir(migrationDir, { - withFileTypes: true, + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + const database = `extractor-${dbIndex++}`; + + const knex = Knex({ + client: WrappedClientPgLite, + dialect: 'postgres', + connection: { + database, + }, }); - const migrationTargets = migrationFiles - .filter(entry => entry.isDirectory()) - .map(entry => entry.name); - if (migrationFiles.some(entry => entry.isFile())) { - migrationTargets.push('.'); - } + await knex.raw(`CREATE DATABASE "${database}"`); - for (const migrationTarget of migrationTargets) { - const database = `extractor-${dbIndex++}`; - - const knex = Knex({ - client: PgLiteClient, - dialect: 'postgres', - connection: { - database, - }, - }); - - await knex.raw(`CREATE DATABASE "${database}"`); - - await runSingleSqlExtraction( - packageDir, - migrationTarget, - pkgName, - knex, - options, - ); - } + await runSingleSqlExtraction( + packageDir, + migrationTarget, + pkgName, + knex, + options, + ); } - } catch (error) { - console.error(error); - process.exit(1); } } diff --git a/packages/repo-tools/src/types.d.ts b/packages/repo-tools/src/types.d.ts deleted file mode 100644 index 2e30114f10..0000000000 --- a/packages/repo-tools/src/types.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2024 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// It's missing a types entry point, but has types in dist -declare module 'embedded-postgres' { - export { default } from 'embedded-postgres/dist/index'; -} diff --git a/yarn.lock b/yarn.lock index 0c67429722..1b70dfda3b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8293,6 +8293,7 @@ __metadata: "@backstage/config-loader": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" + "@electric-sql/pglite": ^0.2.15 "@manypkg/get-packages": ^1.1.3 "@microsoft/api-documenter": ^7.25.7 "@microsoft/api-extractor": ^7.47.2 @@ -8313,13 +8314,13 @@ __metadata: codeowners-utils: ^1.0.2 command-exists: ^1.2.9 commander: ^12.0.0 - embedded-postgres: 17.2.0-beta.15 fs-extra: ^11.2.0 glob: ^8.0.3 is-glob: ^4.0.3 js-yaml: ^4.1.0 just-diff: ^6.0.2 knex: ^3.0.0 + knex-pglite: ^0.11.0 lodash: ^4.17.21 minimatch: ^9.0.0 p-limit: ^3.0.2 @@ -8332,12 +8333,9 @@ __metadata: "@microsoft/tsdoc": "*" "@microsoft/tsdoc-config": "*" "@useoptic/optic": ^1.0.0 - embedded-postgres: 17.2.0-beta.15 prettier: ^2.8.1 typescript: "> 3.0.0" peerDependenciesMeta: - embedded-postgres: - optional: true prettier: optional: true bin: @@ -8964,59 +8962,10 @@ __metadata: languageName: node linkType: hard -"@embedded-postgres/darwin-arm64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/darwin-arm64@npm:17.2.0-beta.15" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - -"@embedded-postgres/darwin-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/darwin-x64@npm:17.2.0-beta.15" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-arm64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-arm64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=arm64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-arm@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-arm@npm:17.2.0-beta.15" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - -"@embedded-postgres/linux-ia32@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-ia32@npm:17.2.0-beta.15" - conditions: os=linux & cpu=ia32 - languageName: node - linkType: hard - -"@embedded-postgres/linux-ppc64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-ppc64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=ppc64 - languageName: node - linkType: hard - -"@embedded-postgres/linux-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/linux-x64@npm:17.2.0-beta.15" - conditions: os=linux & cpu=x64 - languageName: node - linkType: hard - -"@embedded-postgres/windows-x64@npm:^17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "@embedded-postgres/windows-x64@npm:17.2.0-beta.15" - conditions: os=win32 & cpu=x64 +"@electric-sql/pglite@npm:^0.2.14, @electric-sql/pglite@npm:^0.2.15": + version: 0.2.15 + resolution: "@electric-sql/pglite@npm:0.2.15" + checksum: 714136449bdbef92ffc8475ba2237418bbdcef3bdbeb0eb43ab2e9f5fc4c2d9c67c41cbcd1e5ad7ba758d22f1bfb996a3f372745509845e55d9f69349f396aca languageName: node linkType: hard @@ -26898,41 +26847,6 @@ __metadata: languageName: node linkType: hard -"embedded-postgres@npm:17.2.0-beta.15": - version: 17.2.0-beta.15 - resolution: "embedded-postgres@npm:17.2.0-beta.15" - dependencies: - "@embedded-postgres/darwin-arm64": ^17.2.0-beta.15 - "@embedded-postgres/darwin-x64": ^17.2.0-beta.15 - "@embedded-postgres/linux-arm": ^17.2.0-beta.15 - "@embedded-postgres/linux-arm64": ^17.2.0-beta.15 - "@embedded-postgres/linux-ia32": ^17.2.0-beta.15 - "@embedded-postgres/linux-ppc64": ^17.2.0-beta.15 - "@embedded-postgres/linux-x64": ^17.2.0-beta.15 - "@embedded-postgres/windows-x64": ^17.2.0-beta.15 - async-exit-hook: ^2.0.1 - pg: ^8.7.3 - dependenciesMeta: - "@embedded-postgres/darwin-arm64": - optional: true - "@embedded-postgres/darwin-x64": - optional: true - "@embedded-postgres/linux-arm": - optional: true - "@embedded-postgres/linux-arm64": - optional: true - "@embedded-postgres/linux-ia32": - optional: true - "@embedded-postgres/linux-ppc64": - optional: true - "@embedded-postgres/linux-x64": - optional: true - "@embedded-postgres/windows-x64": - optional: true - checksum: 53b261693dcc83cea6cc2589794dbc69bcf4cae508f2021d4686d7fb08e686e93db97392ebd28dfd2bb0126060b54fa609487d9929a4fee19da6d66dc568863f - languageName: node - linkType: hard - "emittery@npm:^0.13.1": version: 0.13.1 resolution: "emittery@npm:0.13.1" @@ -33807,7 +33721,17 @@ __metadata: languageName: node linkType: hard -"knex@npm:3, knex@npm:^3.0.0": +"knex-pglite@npm:^0.11.0": + version: 0.11.0 + resolution: "knex-pglite@npm:0.11.0" + dependencies: + "@electric-sql/pglite": ^0.2.14 + knex: 3.1.0 + checksum: 680de87b9c567bfc3009c81e85584e4e634ede8b98f14acd4d4463e1eae084cdfc0eed05979db742ee2af1bd6f857bb23ca28c4175abdf27b3d6e3e2dfcba4df + languageName: node + linkType: hard + +"knex@npm:3, knex@npm:3.1.0, knex@npm:^3.0.0": version: 3.1.0 resolution: "knex@npm:3.1.0" dependencies: @@ -38487,7 +38411,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": +"pg@npm:^8.11.3, pg@npm:^8.9.0": version: 8.13.1 resolution: "pg@npm:8.13.1" dependencies: