repo-tools: sql extraction schema info

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/repo-tools/package.json

@@ -70,6 +70,7 @@
     "glob": "^8.0.3",
     "is-glob": "^4.0.3",
     "js-yaml": "^4.1.0",
+    "just-diff": "^6.0.2",
     "knex": "^3.0.0",
     "lodash": "^4.17.21",
     "minimatch": "^9.0.0",

packages/repo-tools/src/commands/api-reports/sql-report.ts

@@ -16,6 +16,9 @@
 
 import fs from 'fs-extra';
 import { paths as cliPaths } from '../../lib/paths';
+import { getPortPromise } from 'portfinder';
+import type { Knex } from 'knex';
+import { diff as justDiff } from 'just-diff';
 
 interface SqlExtractionOptions {
   packageDirs: string[];
@@ -62,9 +65,202 @@ async function runSingleSqlExtraction(
 
   console.log(`Extracting SQL from ${migrationDir}`);
 
-  const knex = await import('knex');
-  const EmbeddedPostgres = await import('embedded-postgres');
+  const { default: Knex } = await import('knex');
+  const { default: EmbeddedPostgres } = (await import(
+    'embedded-postgres'
+  )) as typeof import('embedded-postgres/dist/index.d.ts');
 
-  console.log(`DEBUG: knex=`, knex);
+  console.log(`DEBUG: knex=`, Knex);
   console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres);
+
+  const port = await getPortPromise({
+    /*  startPort: 5433, stopPort: 6543  */
+  });
+  console.log(`DEBUG: port=`, port);
+
+  const pg = new EmbeddedPostgres({
+    databaseDir: './data/db',
+    user: 'postgres',
+    password: 'password',
+    port,
+    persistent: false,
+    onError(_messageOrError) {
+      // console.error('EmbeddedPostgres error:', messageOrError);
+    },
+    onLog(_message) {
+      // console.log('EmbeddedPostgres log:', message);
+    },
+  });
+
+  // Create the cluster config files
+  await pg.initialise();
+
+  // Start the server
+  await pg.start();
+
+  await pg.createDatabase('extractor');
+
+  const knex = Knex({
+    client: 'pg',
+    connection: {
+      host: 'localhost',
+      port,
+      user: 'postgres',
+      password: 'password',
+      database: 'extractor',
+    },
+  });
+
+  const migrationsListResult = await knex.migrate.list({
+    directory: migrationDir,
+  });
+  const migrations: string[] = migrationsListResult[1].map(
+    (m: { file: string }) => m.file,
+  );
+
+  const schemaInfoBeforeMigration = new Map<string, SchemaInfo>();
+
+  for (const migration of migrations) {
+    console.log(`DEBUG: UP ${migration}`);
+    const schemaInfo = await getPostgresSchemaInfo(knex);
+    schemaInfoBeforeMigration.set(migration, schemaInfo);
+
+    await knex.migrate.up({
+      directory: migrationDir,
+      name: migration,
+    });
+  }
+
+  const schemaInfo = await getPostgresSchemaInfo(knex);
+  console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2));
+
+  for (const migration of migrations.toReversed()) {
+    console.log(`DEBUG: DOWN ${migration}`);
+    await knex.migrate.down({
+      directory: migrationDir,
+      name: migration,
+    });
+    const after = await getPostgresSchemaInfo(knex);
+    const before = schemaInfoBeforeMigration.get(migration);
+    if (!before) {
+      throw new Error(`No previous result for migration ${migration}`);
+    }
+
+    const diff = justDiff(before, after);
+    console.log(`DEBUG: diff=`, diff);
+    if (diff.length !== 0) {
+      console.log(`Migration ${migration} is not reversible`);
+      await pg.stop();
+      return;
+    }
+  }
+
+  // Stop the server
+  await pg.stop();
+}
+
+type SchemaColumnInfo = {
+  name: string;
+  type: string;
+  nullable: boolean;
+  maxLength: number | null;
+  defaultValue: Knex.Value;
+};
+
+type SchemaIndexInfo = {
+  name: string;
+  unique: boolean;
+  primary: boolean;
+  columns: string[];
+};
+
+type SchemaTableInfo = {
+  name: string;
+  columns: Record<string, SchemaColumnInfo>;
+  indices: Record<string, SchemaIndexInfo>;
+};
+
+type SchemaSequenceInfo = {
+  name: string;
+  type: string;
+};
+
+type SchemaInfo = {
+  tables: Record<string, SchemaTableInfo>;
+  sequences: Record<string, SchemaSequenceInfo>;
+};
+
+async function getPostgresSchemaInfo(knex: Knex): Promise<SchemaInfo> {
+  const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(`
+    SELECT table_name as name
+    FROM information_schema.tables
+    WHERE
+      table_schema = 'public'
+      AND table_type = 'BASE TABLE'
+      AND table_name NOT LIKE 'knex_migrations%'
+  `);
+
+  const tables = Object.fromEntries(
+    await Promise.all(
+      tableNames.map(async ({ name }) => {
+        const columns = await knex.table(name).columnInfo();
+        const { rows: indices } = await knex.raw<{
+          rows: SchemaIndexInfo[];
+        }>(
+          `
+          SELECT
+            index_class.relname as name,
+            index.indisunique as unique,
+            index.indisprimary as primary,
+            json_agg(attribute.attname ORDER BY keys.rn) as columns
+          FROM
+            pg_class table_class,
+            pg_class index_class,
+            pg_index index,
+            UNNEST(index.indkey) WITH ORDINALITY keys(id, rn)
+          INNER JOIN pg_attribute attribute
+            ON attribute.attnum = keys.id
+          WHERE
+            table_class.oid = index.indrelid
+            AND table_class.relkind = 'r'
+            AND table_class.relname = ?
+            AND index_class.oid = index.indexrelid
+            AND attribute.attrelid = table_class.oid
+          GROUP BY index_class.relname, index.indexrelid
+        `,
+          [name],
+        );
+        return [
+          name,
+          {
+            name,
+            columns: Object.entries(columns).map(
+              ([columnName, columnInfo]) => ({
+                ...columnInfo,
+                name: columnName,
+              }),
+            ),
+            indices: Object.fromEntries(
+              indices.map(index => [index.name, index]),
+            ),
+          },
+        ];
+      }),
+    ),
+  );
+
+  const { rows: sequences } = await knex.raw<{
+    rows: SchemaSequenceInfo[];
+  }>(`
+    SELECT sequence_name as name, data_type as type
+    FROM information_schema.sequences
+    WHERE
+      sequence_schema = 'public'
+      AND sequence_name NOT LIKE 'knex_migrations%'
+  `);
+
+  return {
+    tables,
+    sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])),
+  };
 }

yarn.lock

@@ -8318,6 +8318,7 @@ __metadata:
     glob: ^8.0.3
     is-glob: ^4.0.3
     js-yaml: ^4.1.0
+    just-diff: ^6.0.2
     knex: ^3.0.0
     lodash: ^4.17.21
     minimatch: ^9.0.0
@@ -33693,6 +33694,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"just-diff@npm:^6.0.2":
+  version: 6.0.2
+  resolution: "just-diff@npm:6.0.2"
+  checksum: 1a0c7524f640cb88ab013862733e710f840927834208fd3b85cbc5da2ced97acc75e7dcfe493268ac6a6514c51dd8624d2fd9d057050efba3c02b81a6dcb7ff9
+  languageName: node
+  linkType: hard
+
 "just-extend@npm:^6.2.0":
   version: 6.2.0
   resolution: "just-extend@npm:6.2.0"