paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

repo-tools: initial SQL extraction wiring

Browse Source 
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2024-12-18 11:47:41 +01:00
parent 10d45f446f
commit e7272f9dfc
5 changed files with 192 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/repo-tools/package.json
+7 -1
View File
@@ -70,6 +70,7 @@
"glob": "^8.0.3",
"is-glob": "^4.0.3",
"js-yaml": "^4.1.0",
"knex": "^3.0.0",
"lodash": "^4.17.21",
"minimatch": "^9.0.0",
"p-limit": "^3.0.2",
@@ -84,17 +85,22 @@
"@backstage/types": "workspace:^",
"@types/is-glob": "^4.0.2",
"@types/node": "^20.16.0",
"@types/prettier": "^2.0.0"
"@types/prettier": "^2.0.0",
"embedded-postgres": "17.2.0-beta.15"
},
"peerDependencies": {
"@microsoft/api-extractor-model": "*",
"@microsoft/tsdoc": "*",
"@microsoft/tsdoc-config": "*",
"@useoptic/optic": "^1.0.0",
"embedded-postgres": "17.2.0-beta.15",
"prettier": "^2.8.1",
"typescript": "> 3.0.0"
},
"peerDependenciesMeta": {
"embedded-postgres": {
"optional": true
},
"prettier": {
"optional": true
}
packages/repo-tools/src/commands/api-reports/api-extractor.ts
+7 -1
View File
@@ -1218,6 +1218,7 @@ export async function categorizePackageDirs(packageDirs: string[]) {
const dirs = packageDirs.slice();
const tsPackageDirs = new Array<string>();
const cliPackageDirs = new Array<string>();
const sqlPackageDirs = new Array<string>();
await Promise.all(
Array(10)
@@ -1241,6 +1242,11 @@ export async function categorizePackageDirs(packageDirs: string[]) {
if (!role) {
return; // Ignore packages without roles
}
if (
await fs.pathExists(cliPaths.resolveTargetRoot(dir, 'migrations'))
) {
sqlPackageDirs.push(dir);
}
// TODO(Rugvip): Inlined packages are ignored because we can't handle @internal exports
// gracefully, and we don't want to have to mark all exports @public etc.
// It would be good if we could include these packages though.
@@ -1256,7 +1262,7 @@ export async function categorizePackageDirs(packageDirs: string[]) {
}),
);
return { tsPackageDirs, cliPackageDirs };
return { tsPackageDirs, cliPackageDirs, sqlPackageDirs };
}
function parseHelpPage(helpPageContent: string) {
packages/repo-tools/src/commands/api-reports/api-reports.ts
+11 -3
View File
@@ -24,6 +24,7 @@ import {
} from './api-extractor';
import { paths as cliPaths, resolvePackagePaths } from '../../lib/paths';
import { generateTypeDeclarations } from './generateTypeDeclarations';
import { runSqlExtraction } from './sql-report';
type Options = {
ci?: boolean;
@@ -79,9 +80,8 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
await generateTypeDeclarations(tsconfigFilePath);
}
const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs(
selectedPackageDirs,
);
const { tsPackageDirs, cliPackageDirs, sqlPackageDirs } =
await categorizePackageDirs(selectedPackageDirs);
if (tsPackageDirs.length > 0) {
console.log('# Generating package API reports');
@@ -104,6 +104,14 @@ export const buildApiReports = async (paths: string[] = [], opts: Options) => {
});
}
if (sqlPackageDirs.length > 0) {
console.log('# Generating package SQL reports');
await runSqlExtraction({
packageDirs: sqlPackageDirs,
isLocalBuild: !isCiBuild,
});
}
if (isDocsBuild) {
console.log('# Generating package documentation');
await buildDocs({
packages/repo-tools/src/commands/api-reports/sql-report.ts
+70
View File
@@ -0,0 +1,70 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { paths as cliPaths } from '../../lib/paths';
interface SqlExtractionOptions {
packageDirs: string[];
isLocalBuild: boolean;
}
export async function runSqlExtraction(options: SqlExtractionOptions) {
for (const packageDir of options.packageDirs) {
const migrationDir = cliPaths.resolveTargetRoot(packageDir, 'migrations');
if (!(await fs.pathExists(migrationDir))) {
console.log(`No SQL migrations found in ${packageDir}`);
continue;
}
console.log(`Extracting SQL migrations from ${packageDir}`);
const migrationFiles = await fs.readdir(migrationDir, {
withFileTypes: true,
});
const migrationTargets = migrationFiles
.filter(entry => entry.isDirectory())
.map(entry => entry.name);
if (migrationFiles.some(entry => entry.isFile())) {
migrationTargets.push('.');
}
for (const migrationTarget of migrationTargets) {
await runSingleSqlExtraction(packageDir, migrationTarget, options);
}
}
}
async function runSingleSqlExtraction(
targetDir: string,
migrationPath: string,
options: SqlExtractionOptions,
) {
const migrationDir = cliPaths.resolveTargetRoot(
targetDir,
'migrations',
migrationPath,
);
console.log(`Extracting SQL from ${migrationDir}`);
const knex = await import('knex');
const EmbeddedPostgres = await import('embedded-postgres');
console.log(`DEBUG: knex=`, knex);
console.log(`DEBUG: EmbeddedPostgres=`, EmbeddedPostgres);
}
yarn.lock
+97 -1
View File
@@ -8313,10 +8313,12 @@ __metadata:
codeowners-utils: ^1.0.2
command-exists: ^1.2.9
commander: ^12.0.0
embedded-postgres: 17.2.0-beta.15
fs-extra: ^11.2.0
glob: ^8.0.3
is-glob: ^4.0.3
js-yaml: ^4.1.0
knex: ^3.0.0
lodash: ^4.17.21
minimatch: ^9.0.0
p-limit: ^3.0.2
@@ -8329,9 +8331,12 @@ __metadata:
"@microsoft/tsdoc": "*"
"@microsoft/tsdoc-config": "*"
"@useoptic/optic": ^1.0.0
embedded-postgres: 17.2.0-beta.15
prettier: ^2.8.1
typescript: "> 3.0.0"
peerDependenciesMeta:
embedded-postgres:
optional: true
prettier:
optional: true
bin:
@@ -8958,6 +8963,62 @@ __metadata:
languageName: node
linkType: hard
"@embedded-postgres/darwin-arm64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/darwin-arm64@npm:17.2.0-beta.15"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
"@embedded-postgres/darwin-x64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/darwin-x64@npm:17.2.0-beta.15"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
"@embedded-postgres/linux-arm64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/linux-arm64@npm:17.2.0-beta.15"
conditions: os=linux & cpu=arm64
languageName: node
linkType: hard
"@embedded-postgres/linux-arm@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/linux-arm@npm:17.2.0-beta.15"
conditions: os=linux & cpu=arm
languageName: node
linkType: hard
"@embedded-postgres/linux-ia32@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/linux-ia32@npm:17.2.0-beta.15"
conditions: os=linux & cpu=ia32
languageName: node
linkType: hard
"@embedded-postgres/linux-ppc64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/linux-ppc64@npm:17.2.0-beta.15"
conditions: os=linux & cpu=ppc64
languageName: node
linkType: hard
"@embedded-postgres/linux-x64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/linux-x64@npm:17.2.0-beta.15"
conditions: os=linux & cpu=x64
languageName: node
linkType: hard
"@embedded-postgres/windows-x64@npm:^17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "@embedded-postgres/windows-x64@npm:17.2.0-beta.15"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
"@emotion/babel-plugin@npm:^11.13.5":
version: 11.13.5
resolution: "@emotion/babel-plugin@npm:11.13.5"
@@ -26836,6 +26897,41 @@ __metadata:
languageName: node
linkType: hard
"embedded-postgres@npm:17.2.0-beta.15":
version: 17.2.0-beta.15
resolution: "embedded-postgres@npm:17.2.0-beta.15"
dependencies:
"@embedded-postgres/darwin-arm64": ^17.2.0-beta.15
"@embedded-postgres/darwin-x64": ^17.2.0-beta.15
"@embedded-postgres/linux-arm": ^17.2.0-beta.15
"@embedded-postgres/linux-arm64": ^17.2.0-beta.15
"@embedded-postgres/linux-ia32": ^17.2.0-beta.15
"@embedded-postgres/linux-ppc64": ^17.2.0-beta.15
"@embedded-postgres/linux-x64": ^17.2.0-beta.15
"@embedded-postgres/windows-x64": ^17.2.0-beta.15
async-exit-hook: ^2.0.1
pg: ^8.7.3
dependenciesMeta:
"@embedded-postgres/darwin-arm64":
optional: true
"@embedded-postgres/darwin-x64":
optional: true
"@embedded-postgres/linux-arm":
optional: true
"@embedded-postgres/linux-arm64":
optional: true
"@embedded-postgres/linux-ia32":
optional: true
"@embedded-postgres/linux-ppc64":
optional: true
"@embedded-postgres/linux-x64":
optional: true
"@embedded-postgres/windows-x64":
optional: true
checksum: 53b261693dcc83cea6cc2589794dbc69bcf4cae508f2021d4686d7fb08e686e93db97392ebd28dfd2bb0126060b54fa609487d9929a4fee19da6d66dc568863f
languageName: node
linkType: hard
"emittery@npm:^0.13.1":
version: 0.13.1
resolution: "emittery@npm:0.13.1"
@@ -38383,7 +38479,7 @@ __metadata:
languageName: node
linkType: hard
"pg@npm:^8.11.3, pg@npm:^8.9.0":
"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0":
version: 8.13.1
resolution: "pg@npm:8.13.1"
dependencies: