From be2fa90773b94381ebb02fc7e97ad83b601883ec Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Dec 2024 13:33:05 +0100 Subject: [PATCH] repo-tools: refactor sql-reports into split modules Signed-off-by: Patrik Oldsberg --- .../sql-reports/getPgSchemaInfo.ts | 93 ++++++++++++++ .../sql-reports/runSqlExtraction.ts | 119 +----------------- .../commands/api-reports/sql-reports/types.ts | 48 +++++++ packages/repo-tools/src/types.d.ts | 20 +++ 4 files changed, 167 insertions(+), 113 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/types.ts create mode 100644 packages/repo-tools/src/types.d.ts diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts new file mode 100644 index 0000000000..cc8db86ccd --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; +import { SchemaIndexInfo, SchemaInfo, SchemaSequenceInfo } from './types'; + +export async function getPgSchemaInfo(knex: Knex): Promise { + const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` + SELECT table_name as name + FROM information_schema.tables + WHERE + table_schema = 'public' + AND table_type = 'BASE TABLE' + AND table_name NOT LIKE 'knex_migrations%' + `); + + const tables = Object.fromEntries( + await Promise.all( + tableNames.map(async ({ name }) => { + const columns = await knex.table(name).columnInfo(); + const { rows: indices } = await knex.raw<{ + rows: SchemaIndexInfo[]; + }>( + ` + SELECT + index_class.relname as name, + index.indisunique as unique, + index.indisprimary as primary, + json_agg(attribute.attname ORDER BY keys.rn) as columns + FROM + pg_class table_class, + pg_class index_class, + pg_index index, + UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) + INNER JOIN pg_attribute attribute + ON attribute.attnum = keys.id + WHERE + table_class.oid = index.indrelid + AND table_class.relkind = 'r' + AND table_class.relname = ? + AND index_class.oid = index.indexrelid + AND attribute.attrelid = table_class.oid + GROUP BY index_class.relname, index.indexrelid + `, + [name], + ); + return [ + name, + { + name, + columns: Object.entries(columns).map( + ([columnName, columnInfo]) => ({ + ...columnInfo, + name: columnName, + }), + ), + indices: Object.fromEntries( + indices.map(index => [index.name, index]), + ), + }, + ]; + }), + ), + ); + + const { rows: sequences } = await knex.raw<{ + rows: SchemaSequenceInfo[]; + }>(` + SELECT sequence_name as name, data_type as type + FROM information_schema.sequences + WHERE + sequence_schema = 'public' + AND sequence_name NOT LIKE 'knex_migrations%' + `); + + return { + tables, + sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), + }; +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 6fcac744a8..2300fee301 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -17,8 +17,9 @@ import fs from 'fs-extra'; import { paths as cliPaths } from '../../../lib/paths'; import { getPortPromise } from 'portfinder'; -import type { Knex } from 'knex'; import { diff as justDiff } from 'just-diff'; +import { SchemaInfo } from './types'; +import { getPgSchemaInfo } from './getPgSchemaInfo'; interface SqlExtractionOptions { packageDirs: string[]; @@ -66,9 +67,7 @@ async function runSingleSqlExtraction( console.log(`Extracting SQL from ${migrationDir}`); const { default: Knex } = await import('knex'); - const { default: EmbeddedPostgres } = (await import( - 'embedded-postgres' - )) as typeof import('embedded-postgres/dist/index.d.ts'); + const { default: EmbeddedPostgres } = await import('embedded-postgres'); console.log(`DEBUG: knex=`, Knex); console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); @@ -122,7 +121,7 @@ async function runSingleSqlExtraction( for (const migration of migrations) { console.log(`DEBUG: UP ${migration}`); - const schemaInfo = await getPostgresSchemaInfo(knex); + const schemaInfo = await getPgSchemaInfo(knex); schemaInfoBeforeMigration.set(migration, schemaInfo); await knex.migrate.up({ @@ -131,7 +130,7 @@ async function runSingleSqlExtraction( }); } - const schemaInfo = await getPostgresSchemaInfo(knex); + const schemaInfo = await getPgSchemaInfo(knex); console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2)); for (const migration of migrations.toReversed()) { @@ -140,7 +139,7 @@ async function runSingleSqlExtraction( directory: migrationDir, name: migration, }); - const after = await getPostgresSchemaInfo(knex); + const after = await getPgSchemaInfo(knex); const before = schemaInfoBeforeMigration.get(migration); if (!before) { throw new Error(`No previous result for migration ${migration}`); @@ -158,109 +157,3 @@ async function runSingleSqlExtraction( // Stop the server await pg.stop(); } - -type SchemaColumnInfo = { - name: string; - type: string; - nullable: boolean; - maxLength: number | null; - defaultValue: Knex.Value; -}; - -type SchemaIndexInfo = { - name: string; - unique: boolean; - primary: boolean; - columns: string[]; -}; - -type SchemaTableInfo = { - name: string; - columns: Record; - indices: Record; -}; - -type SchemaSequenceInfo = { - name: string; - type: string; -}; - -type SchemaInfo = { - tables: Record; - sequences: Record; -}; - -async function getPostgresSchemaInfo(knex: Knex): Promise { - const { rows: tableNames } = await knex.raw<{ rows: { name: string }[] }>(` - SELECT table_name as name - FROM information_schema.tables - WHERE - table_schema = 'public' - AND table_type = 'BASE TABLE' - AND table_name NOT LIKE 'knex_migrations%' - `); - - const tables = Object.fromEntries( - await Promise.all( - tableNames.map(async ({ name }) => { - const columns = await knex.table(name).columnInfo(); - const { rows: indices } = await knex.raw<{ - rows: SchemaIndexInfo[]; - }>( - ` - SELECT - index_class.relname as name, - index.indisunique as unique, - index.indisprimary as primary, - json_agg(attribute.attname ORDER BY keys.rn) as columns - FROM - pg_class table_class, - pg_class index_class, - pg_index index, - UNNEST(index.indkey) WITH ORDINALITY keys(id, rn) - INNER JOIN pg_attribute attribute - ON attribute.attnum = keys.id - WHERE - table_class.oid = index.indrelid - AND table_class.relkind = 'r' - AND table_class.relname = ? - AND index_class.oid = index.indexrelid - AND attribute.attrelid = table_class.oid - GROUP BY index_class.relname, index.indexrelid - `, - [name], - ); - return [ - name, - { - name, - columns: Object.entries(columns).map( - ([columnName, columnInfo]) => ({ - ...columnInfo, - name: columnName, - }), - ), - indices: Object.fromEntries( - indices.map(index => [index.name, index]), - ), - }, - ]; - }), - ), - ); - - const { rows: sequences } = await knex.raw<{ - rows: SchemaSequenceInfo[]; - }>(` - SELECT sequence_name as name, data_type as type - FROM information_schema.sequences - WHERE - sequence_schema = 'public' - AND sequence_name NOT LIKE 'knex_migrations%' - `); - - return { - tables, - sequences: Object.fromEntries(sequences.map(seq => [seq.name, seq])), - }; -} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts new file mode 100644 index 0000000000..c45d3fa01d --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/types.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; + +export type SchemaColumnInfo = { + name: string; + type: string; + nullable: boolean; + maxLength: number | null; + defaultValue: Knex.Value; +}; + +export type SchemaIndexInfo = { + name: string; + unique: boolean; + primary: boolean; + columns: string[]; +}; + +export type SchemaTableInfo = { + name: string; + columns: Record; + indices: Record; +}; + +export type SchemaSequenceInfo = { + name: string; + type: string; +}; + +export type SchemaInfo = { + tables: Record; + sequences: Record; +}; diff --git a/packages/repo-tools/src/types.d.ts b/packages/repo-tools/src/types.d.ts new file mode 100644 index 0000000000..276b6ec8f9 --- /dev/null +++ b/packages/repo-tools/src/types.d.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// It's missing a types entry point, but has types in dist +declare module 'embedded-postgres' { + export { default } from 'embedded-postgres/dist/index.d.ts'; +}