diff --git a/.changeset/thin-elephants-joke.md b/.changeset/thin-elephants-joke.md new file mode 100644 index 0000000000..a7ed28bc9e --- /dev/null +++ b/.changeset/thin-elephants-joke.md @@ -0,0 +1,6 @@ +--- +'@backstage/cli-module-build': patch +'@backstage/backend-defaults': patch +--- + +Added experimental support for using `embedded-postgres` as the database for local development. Set `backend.database.client` to `embedded-postgres` in your app config to enable this. The `embedded-postgres` package must be installed as an explicit dependency in your project. diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 3d09146b0d..40352c5972 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -585,7 +585,7 @@ export interface Config { /** Database connection configuration, select base database type using the `client` field */ database: { /** Default database client to use */ - client: 'better-sqlite3' | 'sqlite3' | 'pg'; + client: 'better-sqlite3' | 'sqlite3' | 'pg' | 'embedded-postgres'; /** * Base database connection string, or object with individual connection properties * @visibility secret diff --git a/packages/cli-module-build/package.json b/packages/cli-module-build/package.json index 605df9a542..55d3a2558e 100644 --- a/packages/cli-module-build/package.json +++ b/packages/cli-module-build/package.json @@ -77,6 +77,7 @@ "node-stdlib-browser": "^1.3.1", "npm-packlist": "^5.0.0", "p-queue": "^6.6.2", + "portfinder": "^1.0.32", "postcss": "^8.1.0", "postcss-import": "^16.1.0", "process": "^0.11.10", @@ -106,6 +107,15 @@ "@types/fs-extra": "^11.0.0", "@types/lodash": "^4.14.151", "@types/npm-packlist": "^3.0.0", - "@types/shell-quote": "^1.7.5" + "@types/shell-quote": "^1.7.5", + "embedded-postgres": "18.3.0-beta.16" + }, + "peerDependencies": { + "embedded-postgres": "^18.3.0-beta.16" + }, + "peerDependenciesMeta": { + "embedded-postgres": { + "optional": true + } } } diff --git a/packages/cli-module-build/src/commands/package/start/startBackend.ts b/packages/cli-module-build/src/commands/package/start/startBackend.ts index a36a93b8ff..7b71e52da3 100644 --- a/packages/cli-module-build/src/commands/package/start/startBackend.ts +++ b/packages/cli-module-build/src/commands/package/start/startBackend.ts @@ -23,6 +23,7 @@ import { runBackend } from '../../../lib/runner'; interface StartBackendOptions { targetDir: string; checksEnabled: boolean; + configPaths?: string[]; inspectEnabled?: boolean | string; inspectBrkEnabled?: boolean | string; linkedWorkspace?: string; @@ -33,6 +34,7 @@ export async function startBackend(options: StartBackendOptions) { const waitForExit = await runBackend({ targetDir: options.targetDir, entry: 'src/index', + configPaths: options.configPaths, inspectEnabled: options.inspectEnabled, inspectBrkEnabled: options.inspectBrkEnabled, linkedWorkspace: options.linkedWorkspace, @@ -56,6 +58,7 @@ export async function startBackendPlugin(options: StartBackendOptions) { const waitForExit = await runBackend({ targetDir: options.targetDir, entry: 'dev/index', + configPaths: options.configPaths, inspectEnabled: options.inspectEnabled, inspectBrkEnabled: options.inspectBrkEnabled, require: options.require, diff --git a/packages/cli-module-build/src/lib/runner/runBackend.test.ts b/packages/cli-module-build/src/lib/runner/runBackend.test.ts index 2b97ae352c..b6ac8fd54d 100644 --- a/packages/cli-module-build/src/lib/runner/runBackend.test.ts +++ b/packages/cli-module-build/src/lib/runner/runBackend.test.ts @@ -49,6 +49,21 @@ jest.mock('ctrlc-windows', () => ({ ctrlc: jest.fn(), })); +const mockToConfig = jest.fn(); + +jest.mock('@backstage/config-loader', () => ({ + ConfigSources: { + default: () => ({}), + toConfig: (...args: any[]) => mockToConfig(...args), + }, +})); + +const mockStartEmbeddedDb = jest.fn(); + +jest.mock('./startEmbeddedDb', () => ({ + startEmbeddedDb: (...args: any[]) => mockStartEmbeddedDb(...args), +})); + describe('runBackend', () => { let originalEnv: NodeJS.ProcessEnv; let originalPlatform: string; @@ -68,6 +83,12 @@ describe('runBackend', () => { // Mock process.once to prevent actual signal handling jest.spyOn(process, 'once').mockReturnValue(process); + + mockToConfig.mockResolvedValue({ + close: jest.fn(), + getOptionalString: () => undefined, + }); + mockStartEmbeddedDb.mockReset(); }); afterEach(() => { @@ -82,92 +103,73 @@ describe('runBackend', () => { }); describe('--no-node-snapshot argument handling', () => { - it('should pass --no-node-snapshot when NODE_OPTIONS is not set', () => { + it('should pass --no-node-snapshot when NODE_OPTIONS is not set', async () => { delete process.env.NODE_OPTIONS; - runBackend({ - entry: 'src/index', - }); + runBackend({ entry: 'src/index' }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; expect(spawnArgs).toContain('--no-node-snapshot'); }); - it('should pass --no-node-snapshot when NODE_OPTIONS exists without --node-snapshot', () => { + it('should pass --no-node-snapshot when NODE_OPTIONS exists without --node-snapshot', async () => { process.env.NODE_OPTIONS = '--max-old-space-size=4096'; - runBackend({ - entry: 'src/index', - }); + runBackend({ entry: 'src/index' }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; expect(spawnArgs).toContain('--no-node-snapshot'); }); - it('should not pass --no-node-snapshot when --node-snapshot already exists in NODE_OPTIONS', () => { + it('should not pass --no-node-snapshot when --node-snapshot already exists in NODE_OPTIONS', async () => { process.env.NODE_OPTIONS = '--node-snapshot --max-old-space-size=4096'; - runBackend({ - entry: 'src/index', - }); + runBackend({ entry: 'src/index' }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; expect(spawnArgs).not.toContain('--no-node-snapshot'); }); - it('should not pass --no-node-snapshot when --node-snapshot exists in the middle of NODE_OPTIONS', () => { + it('should not pass --no-node-snapshot when --node-snapshot exists in the middle of NODE_OPTIONS', async () => { process.env.NODE_OPTIONS = '--max-old-space-size=4096 --node-snapshot --inspect'; - runBackend({ - entry: 'src/index', - }); + runBackend({ entry: 'src/index' }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; expect(spawnArgs).not.toContain('--no-node-snapshot'); }); - it('should pass --no-node-snapshot even with trailing spaces in NODE_OPTIONS', () => { + it('should pass --no-node-snapshot even with trailing spaces in NODE_OPTIONS', async () => { process.env.NODE_OPTIONS = '--max-old-space-size=4096 '; - runBackend({ - entry: 'src/index', - }); + runBackend({ entry: 'src/index' }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; expect(spawnArgs).toContain('--no-node-snapshot'); }); - it('should pass --no-node-snapshot alongside other option args like --inspect', () => { + it('should pass --no-node-snapshot alongside other option args like --inspect', async () => { delete process.env.NODE_OPTIONS; - runBackend({ - entry: 'src/index', - inspectEnabled: true, - }); + runBackend({ entry: 'src/index', inspectEnabled: true }); - // Fast-forward past the debounce delay (100ms) - jest.advanceTimersByTime(100); + await jest.advanceTimersByTimeAsync(100); expect(mockSpawn).toHaveBeenCalled(); const spawnArgs = mockSpawn.mock.calls[0][1] as string[]; @@ -175,4 +177,62 @@ describe('runBackend', () => { expect(spawnArgs).toContain('--inspect'); }); }); + + describe('embedded-postgres support', () => { + it('should start embedded DB and inject config when database client is embedded-postgres', async () => { + mockToConfig.mockResolvedValue({ + close: jest.fn(), + getOptionalString: (key: string) => + key === 'backend.database.client' ? 'embedded-postgres' : undefined, + }); + mockStartEmbeddedDb.mockResolvedValue({ + connection: { + host: 'localhost', + user: 'postgres', + password: 'password', + port: 5555, + }, + close: jest.fn(), + }); + + runBackend({ entry: 'src/index' }); + await jest.advanceTimersByTimeAsync(100); + + expect(mockStartEmbeddedDb).toHaveBeenCalled(); + expect(mockSpawn).toHaveBeenCalled(); + const spawnEnv = mockSpawn.mock.calls[0][2]?.env as Record< + string, + string + >; + const injected = JSON.parse(spawnEnv.APP_CONFIG_backend_database); + expect(injected).toEqual({ + client: 'pg', + connection: { + host: 'localhost', + user: 'postgres', + password: 'password', + port: 5555, + }, + }); + }); + + it('should not start embedded DB for other database clients', async () => { + mockToConfig.mockResolvedValue({ + close: jest.fn(), + getOptionalString: (key: string) => + key === 'backend.database.client' ? 'better-sqlite3' : undefined, + }); + + runBackend({ entry: 'src/index' }); + await jest.advanceTimersByTimeAsync(100); + + expect(mockStartEmbeddedDb).not.toHaveBeenCalled(); + expect(mockSpawn).toHaveBeenCalled(); + const spawnEnv = mockSpawn.mock.calls[0][2]?.env as Record< + string, + string + >; + expect(spawnEnv.APP_CONFIG_backend_database).toBeUndefined(); + }); + }); }); diff --git a/packages/cli-module-build/src/lib/runner/runBackend.ts b/packages/cli-module-build/src/lib/runner/runBackend.ts index 86c249e8be..464ff10cec 100644 --- a/packages/cli-module-build/src/lib/runner/runBackend.ts +++ b/packages/cli-module-build/src/lib/runner/runBackend.ts @@ -20,10 +20,15 @@ import { ctrlc } from 'ctrlc-windows'; import { IpcServer, ServerDataStore } from '../ipc'; import debounce from 'lodash/debounce'; import { fileURLToPath } from 'node:url'; -import { isAbsolute as isAbsolutePath } from 'node:path'; +import { + isAbsolute as isAbsolutePath, + resolve as resolvePath, +} from 'node:path'; import { targetPaths } from '@backstage/cli-common'; +import { ConfigSources } from '@backstage/config-loader'; import spawn from 'cross-spawn'; +import { startEmbeddedDb } from './startEmbeddedDb'; const loaderArgs = [ '--enable-source-maps', @@ -45,6 +50,8 @@ export type RunBackendOptions = { require?: string | string[]; /** An external linked workspace to override module resolution towards */ linkedWorkspace?: string; + /** Config file paths from --config flags */ + configPaths?: string[]; }; export async function runBackend(options: RunBackendOptions) { @@ -57,6 +64,19 @@ export async function runBackend(options: RunBackendOptions) { const server = new IpcServer(); ServerDataStore.bind(server); + const extraEnv: Record = {}; + + let embeddedDb: Awaited> | undefined; + + const dbClient = await readDatabaseClient(options.configPaths); + if (dbClient === 'embedded-postgres') { + embeddedDb = await startEmbeddedDb(); + extraEnv.APP_CONFIG_backend_database = JSON.stringify({ + client: 'pg', + connection: embeddedDb.connection, + }); + } + let exiting = false; let firstStart = true; let child: ChildProcess | undefined; @@ -134,6 +154,7 @@ export async function runBackend(options: RunBackendOptions) { cwd: options.targetDir, env: { ...process.env, + ...extraEnv, BACKSTAGE_CLI_LINKED_WORKSPACE: options.linkedWorkspace, BACKSTAGE_CLI_CHANNEL: '1', ESBK_TSCONFIG_PATH: targetPaths.resolveRoot('tsconfig.json'), @@ -186,6 +207,7 @@ export async function runBackend(options: RunBackendOptions) { }); } + await embeddedDb?.close(); resolveExitPromise(); } @@ -195,3 +217,24 @@ export async function runBackend(options: RunBackendOptions) { return () => exitPromise; } + +async function readDatabaseClient( + configPaths?: string[], +): Promise { + const rootDir = targetPaths.rootDir; + const source = ConfigSources.default({ + rootDir, + allowMissingDefaultConfig: true, + argv: (configPaths ?? []).flatMap(p => [ + '--config', + isAbsolutePath(p) ? p : resolvePath(rootDir, p), + ]), + }); + + const config = await ConfigSources.toConfig(source); + try { + return config.getOptionalString('backend.database.client'); + } finally { + config.close(); + } +} diff --git a/packages/cli-module-build/src/lib/runner/startEmbeddedDb.ts b/packages/cli-module-build/src/lib/runner/startEmbeddedDb.ts new file mode 100644 index 0000000000..c94a5a7f46 --- /dev/null +++ b/packages/cli-module-build/src/lib/runner/startEmbeddedDb.ts @@ -0,0 +1,116 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'node:os'; +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'node:path'; +import { getPortPromise } from 'portfinder'; +import { ForwardedError } from '@backstage/errors'; +import chalk from 'chalk'; + +const TEMP_DIR_PREFIX = 'backstage-dev-db-'; +const PID_FILE = 'backstage.pid'; + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function cleanStaleDatabases() { + const tmpBase = os.tmpdir(); + const entries = (await fs.readdir(tmpBase)).filter(d => + d.startsWith(TEMP_DIR_PREFIX), + ); + await Promise.all( + entries.map(async d => { + const dir = resolvePath(tmpBase, d); + const raw = await fs + .readFile(resolvePath(dir, PID_FILE), 'utf8') + .catch(() => undefined); + const pid = raw ? Number(raw.trim()) : NaN; + if (!pid || !isProcessAlive(pid)) { + await fs.remove(dir); + } + }), + ); +} + +export async function startEmbeddedDb() { + console.warn( + chalk.yellow( + 'WARNING: Using embedded-postgres for local development is experimental and subject to change', + ), + ); + + const { default: EmbeddedPostgres } = await import('embedded-postgres').catch( + error => { + throw new ForwardedError( + `Failed to load 'embedded-postgres' which is required when using ` + + `'embedded-postgres' as the database client. It must be installed ` + + `as an explicit dependency in your project`, + error, + ); + }, + ); + + await cleanStaleDatabases(); + + const host = 'localhost'; + const user = 'postgres'; + const password = 'password'; + const port = await getPortPromise(); + const tmpDir = await fs.mkdtemp(resolvePath(os.tmpdir(), TEMP_DIR_PREFIX)); + + await fs.writeFile(resolvePath(tmpDir, PID_FILE), String(process.pid)); + + const pg = new EmbeddedPostgres({ + databaseDir: tmpDir, + user, + password, + port, + persistent: false, + onError(messageOrError) { + console.error(`[embedded-postgres]`, messageOrError); + }, + onLog() {}, + }); + + try { + await pg.initialise(); + await pg.start(); + } catch (error) { + await pg.stop().catch(() => {}); + await fs.remove(tmpDir).catch(() => {}); + throw error; + } + + return { + connection: { + host, + user, + password, + port, + }, + async close() { + await pg.stop(); + await fs.remove(tmpDir); + }, + }; +} diff --git a/yarn.lock b/yarn.lock index a681fdee50..18720caa90 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2900,6 +2900,7 @@ __metadata: cross-spawn: "npm:^7.0.3" css-loader: "npm:^6.5.1" ctrlc-windows: "npm:^2.1.0" + embedded-postgres: "npm:18.3.0-beta.16" esbuild-loader: "npm:^4.0.0" eslint-rspack-plugin: "npm:^4.2.1" eslint-webpack-plugin: "npm:^4.2.0" @@ -2912,6 +2913,7 @@ __metadata: node-stdlib-browser: "npm:^1.3.1" npm-packlist: "npm:^5.0.0" p-queue: "npm:^6.6.2" + portfinder: "npm:^1.0.32" postcss: "npm:^8.1.0" postcss-import: "npm:^16.1.0" process: "npm:^0.11.10" @@ -2934,6 +2936,11 @@ __metadata: webpack-dev-server: "npm:^5.0.0" yml-loader: "npm:^2.1.0" yn: "npm:^4.0.0" + peerDependencies: + embedded-postgres: ^18.3.0-beta.16 + peerDependenciesMeta: + embedded-postgres: + optional: true bin: cli-module-build: bin/backstage-cli-module-build languageName: unknown @@ -8556,6 +8563,62 @@ __metadata: languageName: node linkType: hard +"@embedded-postgres/darwin-arm64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/darwin-arm64@npm:18.3.0-beta.16" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@embedded-postgres/darwin-x64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/darwin-x64@npm:18.3.0-beta.16" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-arm64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/linux-arm64@npm:18.3.0-beta.16" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-arm@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/linux-arm@npm:18.3.0-beta.16" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@embedded-postgres/linux-ia32@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/linux-ia32@npm:18.3.0-beta.16" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + +"@embedded-postgres/linux-ppc64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/linux-ppc64@npm:18.3.0-beta.16" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@embedded-postgres/linux-x64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/linux-x64@npm:18.3.0-beta.16" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@embedded-postgres/windows-x64@npm:^18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "@embedded-postgres/windows-x64@npm:18.3.0-beta.16" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@emnapi/core@npm:^1.4.3, @emnapi/core@npm:^1.5.0, @emnapi/core@npm:^1.7.1": version: 1.7.1 resolution: "@emnapi/core@npm:1.7.1" @@ -29365,6 +29428,41 @@ __metadata: languageName: node linkType: hard +"embedded-postgres@npm:18.3.0-beta.16": + version: 18.3.0-beta.16 + resolution: "embedded-postgres@npm:18.3.0-beta.16" + dependencies: + "@embedded-postgres/darwin-arm64": "npm:^18.3.0-beta.16" + "@embedded-postgres/darwin-x64": "npm:^18.3.0-beta.16" + "@embedded-postgres/linux-arm": "npm:^18.3.0-beta.16" + "@embedded-postgres/linux-arm64": "npm:^18.3.0-beta.16" + "@embedded-postgres/linux-ia32": "npm:^18.3.0-beta.16" + "@embedded-postgres/linux-ppc64": "npm:^18.3.0-beta.16" + "@embedded-postgres/linux-x64": "npm:^18.3.0-beta.16" + "@embedded-postgres/windows-x64": "npm:^18.3.0-beta.16" + async-exit-hook: "npm:^2.0.1" + pg: "npm:^8.7.3" + dependenciesMeta: + "@embedded-postgres/darwin-arm64": + optional: true + "@embedded-postgres/darwin-x64": + optional: true + "@embedded-postgres/linux-arm": + optional: true + "@embedded-postgres/linux-arm64": + optional: true + "@embedded-postgres/linux-ia32": + optional: true + "@embedded-postgres/linux-ppc64": + optional: true + "@embedded-postgres/linux-x64": + optional: true + "@embedded-postgres/windows-x64": + optional: true + checksum: 10/13ebdec978559d8d5496df521ec6d6a717a6a3e234a7daa1d3d85e8d050626cde927e5d1d382c70eec219afa721d3c28c26a39023de0a5919feb535470860b47 + languageName: node + linkType: hard + "emittery@npm:^0.13.1": version: 0.13.1 resolution: "emittery@npm:0.13.1" @@ -41564,7 +41662,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.3, pg@npm:^8.9.0": +"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": version: 8.20.0 resolution: "pg@npm:8.20.0" dependencies: