paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #33714 from backstage/rugvip/dev-db

Browse Source 
cli: experimental embedded-postgres support for local dev
This commit is contained in:
Patrik Oldsberg
2026-04-03 21:07:15 +02:00
committed by GitHub
parent 735af5b2d6 c0908fe5b5
commit 205c8c8dc1
8 changed files with 377 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/thin-elephants-joke.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/cli-module-build': patch
'@backstage/backend-defaults': patch
---
Added experimental support for using `embedded-postgres` as the database for local development. Set `backend.database.client` to `embedded-postgres` in your app config to enable this. The `embedded-postgres` package must be installed as an explicit dependency in your project.
packages/backend-defaults/config.d.ts
Vendored
+1 -1
View File
@@ -585,7 +585,7 @@ export interface Config {
/** Database connection configuration, select base database type using the `client` field */
database: {
/** Default database client to use */
client: 'better-sqlite3' | 'sqlite3' | 'pg';
client: 'better-sqlite3' | 'sqlite3' | 'pg' | 'embedded-postgres';
/**
* Base database connection string, or object with individual connection properties
* @visibility secret
packages/cli-module-build/package.json
+11 -1
View File
@@ -77,6 +77,7 @@
"node-stdlib-browser": "^1.3.1",
"npm-packlist": "^5.0.0",
"p-queue": "^6.6.2",
"portfinder": "^1.0.32",
"postcss": "^8.1.0",
"postcss-import": "^16.1.0",
"process": "^0.11.10",
@@ -106,6 +107,15 @@
"@types/fs-extra": "^11.0.0",
"@types/lodash": "^4.14.151",
"@types/npm-packlist": "^3.0.0",
"@types/shell-quote": "^1.7.5"
"@types/shell-quote": "^1.7.5",
"embedded-postgres": "18.3.0-beta.16"
},
"peerDependencies": {
"embedded-postgres": "^18.3.0-beta.16"
},
"peerDependenciesMeta": {
"embedded-postgres": {
"optional": true
}
}
}
packages/cli-module-build/src/commands/package/start/startBackend.ts
+3
View File
@@ -23,6 +23,7 @@ import { runBackend } from '../../../lib/runner';
interface StartBackendOptions {
targetDir: string;
checksEnabled: boolean;
configPaths?: string[];
inspectEnabled?: boolean | string;
inspectBrkEnabled?: boolean | string;
linkedWorkspace?: string;
@@ -33,6 +34,7 @@ export async function startBackend(options: StartBackendOptions) {
const waitForExit = await runBackend({
targetDir: options.targetDir,
entry: 'src/index',
configPaths: options.configPaths,
inspectEnabled: options.inspectEnabled,
inspectBrkEnabled: options.inspectBrkEnabled,
linkedWorkspace: options.linkedWorkspace,
@@ -56,6 +58,7 @@ export async function startBackendPlugin(options: StartBackendOptions) {
const waitForExit = await runBackend({
targetDir: options.targetDir,
entry: 'dev/index',
configPaths: options.configPaths,
inspectEnabled: options.inspectEnabled,
inspectBrkEnabled: options.inspectBrkEnabled,
require: options.require,
packages/cli-module-build/src/lib/runner/runBackend.test.ts
+97 -37
View File
@@ -49,6 +49,21 @@ jest.mock('ctrlc-windows', () => ({
ctrlc: jest.fn(),
}));
const mockToConfig = jest.fn();
jest.mock('@backstage/config-loader', () => ({
ConfigSources: {
default: () => ({}),
toConfig: (...args: any[]) => mockToConfig(...args),
},
}));
const mockStartEmbeddedDb = jest.fn();
jest.mock('./startEmbeddedDb', () => ({
startEmbeddedDb: (...args: any[]) => mockStartEmbeddedDb(...args),
}));
describe('runBackend', () => {
let originalEnv: NodeJS.ProcessEnv;
let originalPlatform: string;
@@ -68,6 +83,12 @@ describe('runBackend', () => {
// Mock process.once to prevent actual signal handling
jest.spyOn(process, 'once').mockReturnValue(process);
mockToConfig.mockResolvedValue({
close: jest.fn(),
getOptionalString: () => undefined,
});
mockStartEmbeddedDb.mockReset();
});
afterEach(() => {
@@ -82,92 +103,73 @@ describe('runBackend', () => {
});
describe('--no-node-snapshot argument handling', () => {
it('should pass --no-node-snapshot when NODE_OPTIONS is not set', () => {
it('should pass --no-node-snapshot when NODE_OPTIONS is not set', async () => {
delete process.env.NODE_OPTIONS;
runBackend({
entry: 'src/index',
});
runBackend({ entry: 'src/index' });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
expect(spawnArgs).toContain('--no-node-snapshot');
});
it('should pass --no-node-snapshot when NODE_OPTIONS exists without --node-snapshot', () => {
it('should pass --no-node-snapshot when NODE_OPTIONS exists without --node-snapshot', async () => {
process.env.NODE_OPTIONS = '--max-old-space-size=4096';
runBackend({
entry: 'src/index',
});
runBackend({ entry: 'src/index' });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
expect(spawnArgs).toContain('--no-node-snapshot');
});
it('should not pass --no-node-snapshot when --node-snapshot already exists in NODE_OPTIONS', () => {
it('should not pass --no-node-snapshot when --node-snapshot already exists in NODE_OPTIONS', async () => {
process.env.NODE_OPTIONS = '--node-snapshot --max-old-space-size=4096';
runBackend({
entry: 'src/index',
});
runBackend({ entry: 'src/index' });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
expect(spawnArgs).not.toContain('--no-node-snapshot');
});
it('should not pass --no-node-snapshot when --node-snapshot exists in the middle of NODE_OPTIONS', () => {
it('should not pass --no-node-snapshot when --node-snapshot exists in the middle of NODE_OPTIONS', async () => {
process.env.NODE_OPTIONS =
'--max-old-space-size=4096 --node-snapshot --inspect';
runBackend({
entry: 'src/index',
});
runBackend({ entry: 'src/index' });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
expect(spawnArgs).not.toContain('--no-node-snapshot');
});
it('should pass --no-node-snapshot even with trailing spaces in NODE_OPTIONS', () => {
it('should pass --no-node-snapshot even with trailing spaces in NODE_OPTIONS', async () => {
process.env.NODE_OPTIONS = '--max-old-space-size=4096 ';
runBackend({
entry: 'src/index',
});
runBackend({ entry: 'src/index' });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
expect(spawnArgs).toContain('--no-node-snapshot');
});
it('should pass --no-node-snapshot alongside other option args like --inspect', () => {
it('should pass --no-node-snapshot alongside other option args like --inspect', async () => {
delete process.env.NODE_OPTIONS;
runBackend({
entry: 'src/index',
inspectEnabled: true,
});
runBackend({ entry: 'src/index', inspectEnabled: true });
// Fast-forward past the debounce delay (100ms)
jest.advanceTimersByTime(100);
await jest.advanceTimersByTimeAsync(100);
expect(mockSpawn).toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls[0][1] as string[];
@@ -175,4 +177,62 @@ describe('runBackend', () => {
expect(spawnArgs).toContain('--inspect');
});
});
describe('embedded-postgres support', () => {
it('should start embedded DB and inject config when database client is embedded-postgres', async () => {
mockToConfig.mockResolvedValue({
close: jest.fn(),
getOptionalString: (key: string) =>
key === 'backend.database.client' ? 'embedded-postgres' : undefined,
});
mockStartEmbeddedDb.mockResolvedValue({
connection: {
host: 'localhost',
user: 'postgres',
password: 'password',
port: 5555,
},
close: jest.fn(),
});
runBackend({ entry: 'src/index' });
await jest.advanceTimersByTimeAsync(100);
expect(mockStartEmbeddedDb).toHaveBeenCalled();
expect(mockSpawn).toHaveBeenCalled();
const spawnEnv = mockSpawn.mock.calls[0][2]?.env as Record<
string,
string
>;
const injected = JSON.parse(spawnEnv.APP_CONFIG_backend_database);
expect(injected).toEqual({
client: 'pg',
connection: {
host: 'localhost',
user: 'postgres',
password: 'password',
port: 5555,
},
});
});
it('should not start embedded DB for other database clients', async () => {
mockToConfig.mockResolvedValue({
close: jest.fn(),
getOptionalString: (key: string) =>
key === 'backend.database.client' ? 'better-sqlite3' : undefined,
});
runBackend({ entry: 'src/index' });
await jest.advanceTimersByTimeAsync(100);
expect(mockStartEmbeddedDb).not.toHaveBeenCalled();
expect(mockSpawn).toHaveBeenCalled();
const spawnEnv = mockSpawn.mock.calls[0][2]?.env as Record<
string,
string
>;
expect(spawnEnv.APP_CONFIG_backend_database).toBeUndefined();
});
});
});
packages/cli-module-build/src/lib/runner/runBackend.ts
+44 -1
View File
@@ -20,10 +20,15 @@ import { ctrlc } from 'ctrlc-windows';
import { IpcServer, ServerDataStore } from '../ipc';
import debounce from 'lodash/debounce';
import { fileURLToPath } from 'node:url';
import { isAbsolute as isAbsolutePath } from 'node:path';
import {
isAbsolute as isAbsolutePath,
resolve as resolvePath,
} from 'node:path';
import { targetPaths } from '@backstage/cli-common';
import { ConfigSources } from '@backstage/config-loader';
import spawn from 'cross-spawn';
import { startEmbeddedDb } from './startEmbeddedDb';
const loaderArgs = [
'--enable-source-maps',
@@ -45,6 +50,8 @@ export type RunBackendOptions = {
require?: string | string[];
/** An external linked workspace to override module resolution towards */
linkedWorkspace?: string;
/** Config file paths from --config flags */
configPaths?: string[];
};
export async function runBackend(options: RunBackendOptions) {
@@ -57,6 +64,19 @@ export async function runBackend(options: RunBackendOptions) {
const server = new IpcServer();
ServerDataStore.bind(server);
const extraEnv: Record<string, string> = {};
let embeddedDb: Awaited<ReturnType<typeof startEmbeddedDb>> | undefined;
const dbClient = await readDatabaseClient(options.configPaths);
if (dbClient === 'embedded-postgres') {
embeddedDb = await startEmbeddedDb();
extraEnv.APP_CONFIG_backend_database = JSON.stringify({
client: 'pg',
connection: embeddedDb.connection,
});
}
let exiting = false;
let firstStart = true;
let child: ChildProcess | undefined;
@@ -134,6 +154,7 @@ export async function runBackend(options: RunBackendOptions) {
cwd: options.targetDir,
env: {
...process.env,
...extraEnv,
BACKSTAGE_CLI_LINKED_WORKSPACE: options.linkedWorkspace,
BACKSTAGE_CLI_CHANNEL: '1',
ESBK_TSCONFIG_PATH: targetPaths.resolveRoot('tsconfig.json'),
@@ -186,6 +207,7 @@ export async function runBackend(options: RunBackendOptions) {
});
}
await embeddedDb?.close();
resolveExitPromise();
}
@@ -195,3 +217,24 @@ export async function runBackend(options: RunBackendOptions) {
return () => exitPromise;
}
async function readDatabaseClient(
configPaths?: string[],
): Promise<string | undefined> {
const rootDir = targetPaths.rootDir;
const source = ConfigSources.default({
rootDir,
allowMissingDefaultConfig: true,
argv: (configPaths ?? []).flatMap(p => [
'--config',
isAbsolutePath(p) ? p : resolvePath(rootDir, p),
]),
});
const config = await ConfigSources.toConfig(source);
try {
return config.getOptionalString('backend.database.client');
} finally {
config.close();
}
}
packages/cli-module-build/src/lib/runner/startEmbeddedDb.ts
+116
View File
@@ -0,0 +1,116 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'node:os';
import fs from 'fs-extra';
import { resolve as resolvePath } from 'node:path';
import { getPortPromise } from 'portfinder';
import { ForwardedError } from '@backstage/errors';
import chalk from 'chalk';
const TEMP_DIR_PREFIX = 'backstage-dev-db-';
const PID_FILE = 'backstage.pid';
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function cleanStaleDatabases() {
const tmpBase = os.tmpdir();
const entries = (await fs.readdir(tmpBase)).filter(d =>
d.startsWith(TEMP_DIR_PREFIX),
);
await Promise.all(
entries.map(async d => {
const dir = resolvePath(tmpBase, d);
const raw = await fs
.readFile(resolvePath(dir, PID_FILE), 'utf8')
.catch(() => undefined);
const pid = raw ? Number(raw.trim()) : NaN;
if (!pid || !isProcessAlive(pid)) {
await fs.remove(dir);
}
}),
);
}
export async function startEmbeddedDb() {
console.warn(
chalk.yellow(
'WARNING: Using embedded-postgres for local development is experimental and subject to change',
),
);
const { default: EmbeddedPostgres } = await import('embedded-postgres').catch(
error => {
throw new ForwardedError(
`Failed to load 'embedded-postgres' which is required when using ` +
`'embedded-postgres' as the database client. It must be installed ` +
`as an explicit dependency in your project`,
error,
);
},
);
await cleanStaleDatabases();
const host = 'localhost';
const user = 'postgres';
const password = 'password';
const port = await getPortPromise();
const tmpDir = await fs.mkdtemp(resolvePath(os.tmpdir(), TEMP_DIR_PREFIX));
await fs.writeFile(resolvePath(tmpDir, PID_FILE), String(process.pid));
const pg = new EmbeddedPostgres({
databaseDir: tmpDir,
user,
password,
port,
persistent: false,
onError(messageOrError) {
console.error(`[embedded-postgres]`, messageOrError);
},
onLog() {},
});
try {
await pg.initialise();
await pg.start();
} catch (error) {
await pg.stop().catch(() => {});
await fs.remove(tmpDir).catch(() => {});
throw error;
}
return {
connection: {
host,
user,
password,
port,
},
async close() {
await pg.stop();
await fs.remove(tmpDir);
},
};
}
yarn.lock
+99 -1
View File
@@ -2900,6 +2900,7 @@ __metadata:
cross-spawn: "npm:^7.0.3"
css-loader: "npm:^6.5.1"
ctrlc-windows: "npm:^2.1.0"
embedded-postgres: "npm:18.3.0-beta.16"
esbuild-loader: "npm:^4.0.0"
eslint-rspack-plugin: "npm:^4.2.1"
eslint-webpack-plugin: "npm:^4.2.0"
@@ -2912,6 +2913,7 @@ __metadata:
node-stdlib-browser: "npm:^1.3.1"
npm-packlist: "npm:^5.0.0"
p-queue: "npm:^6.6.2"
portfinder: "npm:^1.0.32"
postcss: "npm:^8.1.0"
postcss-import: "npm:^16.1.0"
process: "npm:^0.11.10"
@@ -2934,6 +2936,11 @@ __metadata:
webpack-dev-server: "npm:^5.0.0"
yml-loader: "npm:^2.1.0"
yn: "npm:^4.0.0"
peerDependencies:
embedded-postgres: ^18.3.0-beta.16
peerDependenciesMeta:
embedded-postgres:
optional: true
bin:
cli-module-build: bin/backstage-cli-module-build
languageName: unknown
@@ -8556,6 +8563,62 @@ __metadata:
languageName: node
linkType: hard
"@embedded-postgres/darwin-arm64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/darwin-arm64@npm:18.3.0-beta.16"
conditions: os=darwin & cpu=arm64
languageName: node
linkType: hard
"@embedded-postgres/darwin-x64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/darwin-x64@npm:18.3.0-beta.16"
conditions: os=darwin & cpu=x64
languageName: node
linkType: hard
"@embedded-postgres/linux-arm64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/linux-arm64@npm:18.3.0-beta.16"
conditions: os=linux & cpu=arm64
languageName: node
linkType: hard
"@embedded-postgres/linux-arm@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/linux-arm@npm:18.3.0-beta.16"
conditions: os=linux & cpu=arm
languageName: node
linkType: hard
"@embedded-postgres/linux-ia32@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/linux-ia32@npm:18.3.0-beta.16"
conditions: os=linux & cpu=ia32
languageName: node
linkType: hard
"@embedded-postgres/linux-ppc64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/linux-ppc64@npm:18.3.0-beta.16"
conditions: os=linux & cpu=ppc64
languageName: node
linkType: hard
"@embedded-postgres/linux-x64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/linux-x64@npm:18.3.0-beta.16"
conditions: os=linux & cpu=x64
languageName: node
linkType: hard
"@embedded-postgres/windows-x64@npm:^18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "@embedded-postgres/windows-x64@npm:18.3.0-beta.16"
conditions: os=win32 & cpu=x64
languageName: node
linkType: hard
"@emnapi/core@npm:^1.4.3, @emnapi/core@npm:^1.5.0, @emnapi/core@npm:^1.7.1":
version: 1.7.1
resolution: "@emnapi/core@npm:1.7.1"
@@ -29365,6 +29428,41 @@ __metadata:
languageName: node
linkType: hard
"embedded-postgres@npm:18.3.0-beta.16":
version: 18.3.0-beta.16
resolution: "embedded-postgres@npm:18.3.0-beta.16"
dependencies:
"@embedded-postgres/darwin-arm64": "npm:^18.3.0-beta.16"
"@embedded-postgres/darwin-x64": "npm:^18.3.0-beta.16"
"@embedded-postgres/linux-arm": "npm:^18.3.0-beta.16"
"@embedded-postgres/linux-arm64": "npm:^18.3.0-beta.16"
"@embedded-postgres/linux-ia32": "npm:^18.3.0-beta.16"
"@embedded-postgres/linux-ppc64": "npm:^18.3.0-beta.16"
"@embedded-postgres/linux-x64": "npm:^18.3.0-beta.16"
"@embedded-postgres/windows-x64": "npm:^18.3.0-beta.16"
async-exit-hook: "npm:^2.0.1"
pg: "npm:^8.7.3"
dependenciesMeta:
"@embedded-postgres/darwin-arm64":
optional: true
"@embedded-postgres/darwin-x64":
optional: true
"@embedded-postgres/linux-arm":
optional: true
"@embedded-postgres/linux-arm64":
optional: true
"@embedded-postgres/linux-ia32":
optional: true
"@embedded-postgres/linux-ppc64":
optional: true
"@embedded-postgres/linux-x64":
optional: true
"@embedded-postgres/windows-x64":
optional: true
checksum: 10/13ebdec978559d8d5496df521ec6d6a717a6a3e234a7daa1d3d85e8d050626cde927e5d1d382c70eec219afa721d3c28c26a39023de0a5919feb535470860b47
languageName: node
linkType: hard
"emittery@npm:^0.13.1":
version: 0.13.1
resolution: "emittery@npm:0.13.1"
@@ -41564,7 +41662,7 @@ __metadata:
languageName: node
linkType: hard
"pg@npm:^8.11.3, pg@npm:^8.9.0":
"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0":
version: 8.20.0
resolution: "pg@npm:8.20.0"
dependencies: