chore: pg-lite experiment

Signed-off-by: blam <ben@blam.sh>

packages/repo-tools/package.json

@@ -49,6 +49,7 @@
     "@backstage/cli-node": "workspace:^",
     "@backstage/config-loader": "workspace:^",
     "@backstage/errors": "workspace:^",
+    "@electric-sql/pglite": "^0.2.15",
     "@manypkg/get-packages": "^1.1.3",
     "@microsoft/api-documenter": "^7.25.7",
     "@microsoft/api-extractor": "^7.47.2",
@@ -72,6 +73,7 @@
     "js-yaml": "^4.1.0",
     "just-diff": "^6.0.2",
     "knex": "^3.0.0",
+    "knex-pglite": "^0.11.0",
     "lodash": "^4.17.21",
     "minimatch": "^9.0.0",
     "p-limit": "^3.0.2",

packages/repo-tools/src/commands/api-reports/sql-reports/pglite.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PGlite } from '@electric-sql/pglite';
+import { Client, Knex } from 'knex';
+
+const Client_PG: any = require('knex/lib/dialects/postgres/index.js');
+
+type KnexPGliteConfig = Knex.Config & { connection: { pglite?: PGlite } };
+
+export class PgLiteClient extends Client_PG {
+  private pglite: PGlite | undefined;
+
+  constructor(config: KnexPGliteConfig) {
+    super({
+      ...config,
+      client: 'pg',
+      // Enforce a single connection:
+      pool: { min: 1, max: 1 },
+    } satisfies Knex.Config);
+    if (config.pool) {
+      throw new Error(
+        'PGlite is single user/connection. Pool cannot be configured.',
+      );
+    }
+    this.pglite =
+      config.connection?.pglite ??
+      new PGlite(
+        config.connection?.['filename'] ??
+          config.connection?.['connectionString'],
+      );
+  }
+
+  _driver() {}
+
+  async _acquireOnlyConnection() {
+    const connection = this.pglite;
+    await connection.waitReady;
+    return connection;
+  }
+
+  async destroyRawConnection(connection: PGlite) {
+    // There is only one connection, if this one goes shut down the database
+    await connection.close();
+  }
+
+  async setSchemaSearchPath(
+    connection: PGlite,
+    searchPath: string,
+  ): Promise<boolean> {
+    let path = searchPath || this.searchPath;
+
+    if (!path) {
+      return true;
+    }
+
+    if (!Array.isArray(path) && typeof path !== 'string') {
+      throw new TypeError(
+        `knex: Expected searchPath to be Array/String, got: ${typeof path}`,
+      );
+    }
+
+    if (typeof path === 'string') {
+      if (path.includes(',')) {
+        const parts = path.split(',');
+        const arraySyntax = `[${parts
+          .map(searchPath => `'${searchPath}'`)
+          .join(', ')}]`;
+        this.logger.warn?.(
+          `Detected comma in searchPath "${path}".` +
+            `If you are trying to specify multiple schemas, use Array syntax: ${arraySyntax}`,
+        );
+      }
+      path = [path];
+    }
+
+    path = path.map((schemaName: string) => `"${schemaName}"`).join(',');
+
+    await connection.query(`set search_path to ${path}`);
+    return true;
+  }
+
+  async checkVersion(connection: PGlite) {
+    const resp = await connection.query('select version();');
+    return this._parseVersion((resp.rows[0] as any).version);
+  }
+
+  async _query(connection: PGlite, obj: any) {
+    if (!obj.sql) throw new Error('The query is empty');
+
+    const response = await connection.query(obj.sql, obj.bindings, obj.options);
+    obj.response = response;
+    return obj;
+  }
+
+  processResponse(obj: any, runner: any) {
+    const response = {
+      ...obj.response,
+      rowCount: obj.response.affectedRows,
+      command: (obj.method as string)?.toUpperCase() ?? '',
+    };
+    return super.processResponse({ ...obj, response }, runner);
+  }
+
+  _stream(connection: PGlite, obj: any, stream: any) {
+    return new Promise((resolver, rejecter) => {
+      stream.on('error', rejecter);
+      stream.on('end', resolver);
+
+      return this._query(connection, obj)
+        .then(obj => obj.response.rows)
+        .then(rows => rows.forEach((row: any) => stream.write(row)))
+        .catch(err => {
+          stream.emit('error', err);
+        })
+        .then(() => {
+          stream.end();
+        });
+    });
+  }
+}
+
+Object.assign(PgLiteClient.prototype, {
+  // The "dialect", for reference .
+  driverName: 'postgres',
+});

packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts

@@ -24,6 +24,7 @@ import { getPgSchemaInfo } from './getPgSchemaInfo';
 import { generateSqlReport } from './generateSqlReport';
 import type { Knex } from 'knex';
 import { logApiReportInstructions } from '../api-extractor';
+import { PgLiteClient } from './pglite';
 
 interface SqlExtractionOptions {
   packageDirs: string[];
@@ -32,36 +33,6 @@ interface SqlExtractionOptions {
 
 export async function runSqlExtraction(options: SqlExtractionOptions) {
   const { default: Knex } = await import('knex');
-  const { default: EmbeddedPostgres } = await import('embedded-postgres').catch(
-    error => {
-      throw new Error(
-        `Failed to load peer dependency 'embedded-postgres' for generating SQL reports. ` +
-          `It must be installed as an explicit dependency in your project. Caused by; ${error}`,
-      );
-    },
-  );
-
-  const port = await getPortPromise();
-
-  const basePgOpts = {
-    host: 'localhost',
-    user: 'postgres',
-    password: 'password',
-  };
-  const pg = new EmbeddedPostgres({
-    databaseDir: './data/db',
-    ...basePgOpts,
-    port,
-    persistent: false,
-    onError(_messageOrError) {},
-    onLog(_message) {},
-  });
-
-  // Create the cluster config files
-  await pg.initialise();
-
-  // Start the server
-  await pg.start();
 
   let dbIndex = 1;
 
@@ -90,16 +61,17 @@ export async function runSqlExtraction(options: SqlExtractionOptions) {
 
       for (const migrationTarget of migrationTargets) {
         const database = `extractor-${dbIndex++}`;
-        await pg.createDatabase(database);
 
         const knex = Knex({
-          client: 'pg',
+          client: PgLiteClient,
+          dialect: 'postgres',
           connection: {
-            ...basePgOpts,
-            port,
             database,
           },
         });
+
+        await knex.raw(`CREATE DATABASE "${database}"`);
+
         await runSingleSqlExtraction(
           packageDir,
           migrationTarget,
@@ -109,9 +81,9 @@ export async function runSqlExtraction(options: SqlExtractionOptions) {
         );
       }
     }
-  } finally {
-    // Stop the server
-    await pg.stop();
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
   }
 }