From dee5cf2f3c49ef3e565a8ea4c7c42a17980f1f1c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 28 Dec 2024 16:25:48 +0100 Subject: [PATCH] repo-tools: full SQL report implementation Signed-off-by: Patrik Oldsberg --- .../src/commands/api-reports/api-extractor.ts | 2 +- .../sql-reports/generateSqlReport.ts | 87 ++++++++ .../sql-reports/getPgSchemaInfo.ts | 7 +- .../sql-reports/runSqlExtraction.ts | 199 ++++++++++++------ 4 files changed, 219 insertions(+), 76 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index dde3a33b54..4db8077501 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -284,7 +284,7 @@ export async function getTsDocConfig() { return tsdocConfigFile; } -function logApiReportInstructions() { +export function logApiReportInstructions() { console.log(''); console.log( '*************************************************************************************', diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts new file mode 100644 index 0000000000..53165590eb --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/generateSqlReport.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SchemaInfo } from './types'; + +export function generateSqlReport(options: { + reportName: string; + failedDownMigration?: string; + schemaInfo: SchemaInfo; +}): string { + const { reportName, failedDownMigration, schemaInfo } = options; + + const output = [ + `## SQL Report file for "${reportName}"`, + '', + '> Do not edit this file. It is a report generated by `yarn build:api-reports`', + '', + ]; + + if (failedDownMigration) { + output.push('> [!WARNING]'); + output.push(`> Failed to migrate down from '${failedDownMigration}'`); + output.push(''); + } + + if (Object.keys(schemaInfo.sequences).length > 0) { + output.push('## Sequences'); + output.push(''); + for (const [sequenceName, sequenceInfo] of Object.entries( + schemaInfo.sequences, + )) { + output.push(`- \`${sequenceName}\` (${sequenceInfo.type})`); + } + output.push(''); + } + + for (const [tableName, tableInfo] of Object.entries(schemaInfo.tables)) { + output.push(`## Table \`${tableName}\``); + output.push(''); + output.push(' | Column | Type | Nullable | Max Length | Default |'); + output.push(' |--------|------|----------|------------|---------|'); + for (const [columnName, columnInfo] of Object.entries(tableInfo.columns)) { + output.push( + ` | \`${columnName}\` | ${columnInfo.type} | ${ + columnInfo.nullable + } | ${columnInfo.maxLength ?? '-'} | ${ + columnInfo.defaultValue ?? '-' + } |`, + ); + } + output.push(''); + + if (Object.keys(tableInfo.indices).length > 0) { + output.push('### Indices'); + output.push(''); + for (const [indexName, indexInfo] of Object.entries(tableInfo.indices)) { + const indexType = [ + indexInfo.unique && 'unique', + indexInfo.primary && 'primary', + ] + .filter(Boolean) + .join(' '); + output.push( + `- \`${indexName}\` (\`${indexInfo.columns.join('`, `')}\`)${ + indexType ? ` ${indexType}` : '' + }`, + ); + } + output.push(''); + } + } + + return output.join('\n'); +} diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts index cc8db86ccd..fc0da743ae 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/getPgSchemaInfo.ts @@ -61,12 +61,7 @@ export async function getPgSchemaInfo(knex: Knex): Promise { name, { name, - columns: Object.entries(columns).map( - ([columnName, columnInfo]) => ({ - ...columnInfo, - name: columnName, - }), - ), + columns, indices: Object.fromEntries( indices.map(index => [index.name, index]), ), diff --git a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts index 2300fee301..9a78907cfd 100644 --- a/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts +++ b/packages/repo-tools/src/commands/api-reports/sql-reports/runSqlExtraction.ts @@ -14,12 +14,16 @@ * limitations under the License. */ -import fs from 'fs-extra'; +import fs, { readJson } from 'fs-extra'; +import { relative as relativePath } from 'path'; import { paths as cliPaths } from '../../../lib/paths'; import { getPortPromise } from 'portfinder'; import { diff as justDiff } from 'just-diff'; import { SchemaInfo } from './types'; import { getPgSchemaInfo } from './getPgSchemaInfo'; +import { generateSqlReport } from './generateSqlReport'; +import type { Knex } from 'knex'; +import { logApiReportInstructions } from '../api-extractor'; interface SqlExtractionOptions { packageDirs: string[]; @@ -27,60 +31,19 @@ interface SqlExtractionOptions { } export async function runSqlExtraction(options: SqlExtractionOptions) { - for (const packageDir of options.packageDirs) { - const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); - if (!(await fs.pathExists(migrationDir))) { - console.log(`No SQL migrations found in ${packageDir}`); - continue; - } - - console.log(`Extracting SQL migrations from ${packageDir}`); - - const migrationFiles = await fs.readdir(migrationDir, { - withFileTypes: true, - }); - - const migrationTargets = migrationFiles - .filter(entry => entry.isDirectory()) - .map(entry => entry.name); - if (migrationFiles.some(entry => entry.isFile())) { - migrationTargets.push('.'); - } - - for (const migrationTarget of migrationTargets) { - await runSingleSqlExtraction(packageDir, migrationTarget, options); - } - } -} - -async function runSingleSqlExtraction( - targetDir: string, - migrationPath: string, - options: SqlExtractionOptions, -) { - const migrationDir = cliPaths.resolveTargetRoot( - targetDir, - 'migrations', - migrationPath, - ); - - console.log(`Extracting SQL from ${migrationDir}`); - const { default: Knex } = await import('knex'); const { default: EmbeddedPostgres } = await import('embedded-postgres'); - console.log(`DEBUG: knex=`, Knex); - console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres); + const port = await getPortPromise(); - const port = await getPortPromise({ - /* startPort: 5433, stopPort: 6543 */ - }); - console.log(`DEBUG: port=`, port); - - const pg = new EmbeddedPostgres({ - databaseDir: './data/db', + const basePgOpts = { + host: 'localhost', user: 'postgres', password: 'password', + }; + const pg = new EmbeddedPostgres({ + databaseDir: './data/db', + ...basePgOpts, port, persistent: false, onError(_messageOrError) { @@ -97,18 +60,75 @@ async function runSingleSqlExtraction( // Start the server await pg.start(); - await pg.createDatabase('extractor'); + let dbIndex = 1; - const knex = Knex({ - client: 'pg', - connection: { - host: 'localhost', - port, - user: 'postgres', - password: 'password', - database: 'extractor', - }, - }); + try { + for (const packageDir of options.packageDirs) { + const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations'); + if (!(await fs.pathExists(migrationDir))) { + console.log(`No SQL migrations found in ${packageDir}`); + continue; + } + + const { name: pkgName } = await readJson( + cliPaths.resolveTargetRoot(packageDir, 'package.json'), + ); + + const migrationFiles = await fs.readdir(migrationDir, { + withFileTypes: true, + }); + + const migrationTargets = migrationFiles + .filter(entry => entry.isDirectory()) + .map(entry => entry.name); + if (migrationFiles.some(entry => entry.isFile())) { + migrationTargets.push('.'); + } + + for (const migrationTarget of migrationTargets) { + const database = `extractor-${dbIndex++}`; + await pg.createDatabase(database); + + const knex = Knex({ + client: 'pg', + connection: { + ...basePgOpts, + port, + database, + }, + }); + await runSingleSqlExtraction( + packageDir, + migrationTarget, + pkgName, + knex, + options, + ); + } + } + } finally { + // Stop the server + await pg.stop(); + } +} + +async function runSingleSqlExtraction( + targetDir: string, + migrationTarget: string, + pkgName: string, + knex: Knex, + options: SqlExtractionOptions, +) { + const migrationDir = cliPaths.resolveTargetRoot( + targetDir, + 'migrations', + migrationTarget, + ); + + const reportName = + migrationTarget === '.' ? pkgName : `${pkgName}/${migrationTarget}`; + + console.log(`Generating SQL report for ${reportName}`); const migrationsListResult = await knex.migrate.list({ directory: migrationDir, @@ -120,7 +140,6 @@ async function runSingleSqlExtraction( const schemaInfoBeforeMigration = new Map(); for (const migration of migrations) { - console.log(`DEBUG: UP ${migration}`); const schemaInfo = await getPgSchemaInfo(knex); schemaInfoBeforeMigration.set(migration, schemaInfo); @@ -131,10 +150,9 @@ async function runSingleSqlExtraction( } const schemaInfo = await getPgSchemaInfo(knex); - console.log(`DEBUG: schemaInfo=`, JSON.stringify(schemaInfo, null, 2)); + let failedDownMigration: string | undefined = undefined; for (const migration of migrations.toReversed()) { - console.log(`DEBUG: DOWN ${migration}`); await knex.migrate.down({ directory: migrationDir, name: migration, @@ -146,14 +164,57 @@ async function runSingleSqlExtraction( } const diff = justDiff(before, after); - console.log(`DEBUG: diff=`, diff); if (diff.length !== 0) { - console.log(`Migration ${migration} is not reversible`); - await pg.stop(); - return; + console.log( + `Migration ${migration} is not reversible: ${JSON.stringify( + diff, + null, + 2, + )}`, + ); + failedDownMigration = migration; + break; } } - // Stop the server - await pg.stop(); + const report = generateSqlReport({ + reportName, + failedDownMigration, + schemaInfo, + }); + + const reportPath = cliPaths.resolveTargetRoot( + targetDir, + `report${migrationTarget === '.' ? '' : `-${migrationTarget}`}.sql.md`, + ); + const existingReport = await fs.readFile(reportPath, 'utf8').catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (existingReport !== report) { + if (options.isLocalBuild) { + console.warn(`SQL report changed for ${targetDir}`); + await fs.writeFile(reportPath, report); + } else { + logApiReportInstructions(); + + if (existingReport) { + console.log(''); + console.log( + `The conflicting file is ${relativePath( + cliPaths.targetRoot, + reportPath, + )}, expecting the following content:`, + ); + console.log(''); + + console.log(report); + + logApiReportInstructions(); + } + throw new Error(`Report ${reportPath} is out of date`); + } + } }