diff --git a/.changeset/green-adults-retire.md b/.changeset/green-adults-retire.md new file mode 100644 index 0000000000..c13e8d664f --- /dev/null +++ b/.changeset/green-adults-retire.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-dev-utils': minor +--- + +Introduced a new package for backend development utilities. Similar to how `@backstage/dev-utils` is used in the frontend. diff --git a/.changeset/long-chefs-burn.md b/.changeset/long-chefs-burn.md new file mode 100644 index 0000000000..03a7b76db9 --- /dev/null +++ b/.changeset/long-chefs-burn.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Updated database factory to pass service deps required for restoring database state during development. diff --git a/.changeset/old-ways-know.md b/.changeset/old-ways-know.md new file mode 100644 index 0000000000..60a1a923e8 --- /dev/null +++ b/.changeset/old-ways-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +The `DatabaseManager.forPlugin` method now accepts additional service dependencies. There is no need to update existing code to pass these dependencies. diff --git a/.changeset/red-kiwis-compare.md b/.changeset/red-kiwis-compare.md new file mode 100644 index 0000000000..6045367105 --- /dev/null +++ b/.changeset/red-kiwis-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +The shutdown signal handlers are now installed as part of the backend instance rather than the lifecycle service, and explicitly cause the process to exit. diff --git a/.changeset/weak-radios-sing.md b/.changeset/weak-radios-sing.md new file mode 100644 index 0000000000..dff287b565 --- /dev/null +++ b/.changeset/weak-radios-sing.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Added an experimental mode for the `package start` command for backend packages. Enabled by setting `EXPERIMENTAL_BACKEND_START`. diff --git a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts index 8baf84cb69..4d95f863db 100644 --- a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts +++ b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts @@ -26,7 +26,8 @@ export const databaseFactory = createServiceFactory({ service: coreServices.database, deps: { config: coreServices.config, - plugin: coreServices.pluginMetadata, + lifecycle: coreServices.lifecycle, + pluginMetadata: coreServices.pluginMetadata, }, async createRootContext({ config }) { return config.getOptional('backend.database') @@ -39,7 +40,10 @@ export const databaseFactory = createServiceFactory({ }), ); }, - async factory({ plugin }, databaseManager) { - return databaseManager.forPlugin(plugin.getId()); + async factory({ pluginMetadata, lifecycle }, databaseManager) { + return databaseManager.forPlugin(pluginMetadata.getId(), { + pluginMetadata, + lifecycle, + }); }, }); diff --git a/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts b/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts index 71ab76ae92..ebff53e9a6 100644 --- a/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts +++ b/packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts @@ -22,11 +22,8 @@ import { LoggerService, } from '@backstage/backend-plugin-api'; -const CALLBACKS = ['SIGTERM', 'SIGINT', 'beforeExit']; export class BackendLifecycleImpl implements RootLifecycleService { - constructor(private readonly logger: LoggerService) { - CALLBACKS.map(signal => process.on(signal, () => this.shutdown())); - } + constructor(private readonly logger: LoggerService) {} #isCalled = false; #shutdownTasks: Array = []; diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index ce17398a79..4bdee8b085 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -28,7 +28,7 @@ import { } from './types'; export class BackendInitializer { - #started = false; + #startPromise?: Promise; #features = new Map(); #registerInits = new Array(); #extensionPoints = new Map, unknown>(); @@ -75,18 +75,40 @@ export class BackendInitializer { } add(feature: BackendFeature, options?: TOptions) { - if (this.#started) { + if (this.#startPromise) { throw new Error('feature can not be added after the backend has started'); } this.#features.set(feature, options); } async start(): Promise { - if (this.#started) { + if (this.#startPromise) { throw new Error('Backend has already started'); } - this.#started = true; + const exitHandler = async () => { + process.removeListener('SIGTERM', exitHandler); + process.removeListener('SIGINT', exitHandler); + process.removeListener('beforeExit', exitHandler); + + try { + await this.stop(); + process.exit(0); + } catch (error) { + console.error(error); + process.exit(1); + } + }; + + process.addListener('SIGTERM', exitHandler); + process.addListener('SIGINT', exitHandler); + process.addListener('beforeExit', exitHandler); + + this.#startPromise = this.#doStart(); + await this.#startPromise; + } + + async #doStart(): Promise { // Initialize all root scoped services for (const ref of this.#serviceHolder.getServiceRefs()) { if (ref.scope === 'root') { @@ -177,9 +199,10 @@ export class BackendInitializer { } async stop(): Promise { - if (!this.#started) { + if (!this.#startPromise) { return; } + await this.#startPromise; const lifecycleService = await this.#serviceHolder.get( coreServices.rootLifecycle, diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index b03d6ec866..e77a58b34c 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -32,6 +32,7 @@ import { IdentityService } from '@backstage/backend-plugin-api'; import { isChildPath } from '@backstage/cli-common'; import { Knex } from 'knex'; import { KubeConfig } from '@kubernetes/client-node'; +import { LifecycleService } from '@backstage/backend-plugin-api'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { Logger } from 'winston'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -40,6 +41,7 @@ import { PermissionsService } from '@backstage/backend-plugin-api'; import { CacheService as PluginCacheManager } from '@backstage/backend-plugin-api'; import { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; import { DiscoveryService as PluginEndpointDiscovery } from '@backstage/backend-plugin-api'; +import { PluginMetadataService } from '@backstage/backend-plugin-api'; import { PushResult } from 'isomorphic-git'; import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; @@ -232,6 +234,10 @@ export class Contexts { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ): Knex; // @public @@ -252,7 +258,13 @@ export function createStatusCheckRouter(options: { // @public export class DatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; + forPlugin( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): PluginDatabaseManager; static fromConfig( config: Config, options?: DatabaseManagerOptions, diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c950950680..9f1069f322 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -35,6 +35,7 @@ }, "dependencies": { "@backstage/backend-app-api": "workspace:^", + "@backstage/backend-dev-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 5a497fff29..08979a92f8 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -29,7 +29,11 @@ import { } from './connection'; import { PluginDatabaseManager } from './types'; import path from 'path'; -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + LifecycleService, + LoggerService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; /** @@ -94,12 +98,18 @@ export class DatabaseManager { * should be unique as they are used to look up database config overrides under * `backend.database.plugin`. */ - forPlugin(pluginId: string): PluginDatabaseManager { + forPlugin( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): PluginDatabaseManager { const _this = this; return { getClient(): Promise { - return _this.getDatabase(pluginId); + return _this.getDatabase(pluginId, deps); }, migrations: { skip: false, @@ -307,7 +317,13 @@ export class DatabaseManager { * @returns Promise which resolves to a scoped Knex database client for a * plugin */ - private async getDatabase(pluginId: string): Promise { + private async getDatabase( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise { if (this.databaseCache.has(pluginId)) { return this.databaseCache.get(pluginId)!; } @@ -351,6 +367,7 @@ export class DatabaseManager { const client = createDatabaseClient( pluginConfig, databaseClientOverrides, + deps, ); this.startKeepaliveLoop(pluginId, client); return client; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 5f487886b4..ff10d349dd 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -22,6 +22,10 @@ import { mergeDatabaseConfig } from './config'; import { DatabaseConnector } from './types'; import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; type DatabaseClient = | 'pg' @@ -55,11 +59,15 @@ const ConnectorMapping: Record = { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ) { const client: DatabaseClient = dbConfig.getString('client'); return ( - ConnectorMapping[client]?.createClient(dbConfig, overrides) ?? + ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ?? knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) ); } diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 6426005f0e..e1ed6b18e9 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -18,8 +18,13 @@ import { Config } from '@backstage/config'; import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; +import { DevDataStore } from '@backstage/backend-dev-utils'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; /** * Creates a knex SQLite3 database connection @@ -30,22 +35,65 @@ import { DatabaseConnector } from '../types'; export function createSqliteDatabaseClient( dbConfig: Config, overrides?: Knex.Config, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); + const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig; + + const filename = connConfig.filename ?? ':memory:'; // If storage on disk is used, ensure that the directory exists - if ( - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename && - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !== - ':memory:' - ) { - const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; + if (filename !== ':memory:') { const directory = path.dirname(filename); - ensureDirSync(directory); } - const database = knexFactory(knexConfig); + let database: Knex; + + if (deps && filename === ':memory:') { + // The dev store is used during watch mode to store and restore the database + // across reloads. It is only available when running the backend through + // `backstage-cli package start`. + const devStore = DevDataStore.get(); + + if (devStore) { + const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`; + + const connectionLoader = async () => { + // If seed data is available, use it tconnectionLoader restore the database + const { data: seedData } = await devStore.load(dataKey); + + return { + ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig), + filename: seedData ?? ':memory:', + }; + }; + + database = knexFactory({ + ...knexConfig, + connection: Object.assign(connectionLoader, { + // This is a workaround for the knex SQLite driver always warning when using a config loader + filename: ':memory:', + }), + }); + + // If the dev store is available we save the database state on shutdown + deps.lifecycle.addShutdownHook({ + async fn() { + const connection = await database.client.acquireConnection(); + const data = connection.serialize(); + await devStore.save(dataKey, data); + }, + }); + } else { + database = knexFactory(knexConfig); + } + } else { + database = knexFactory(knexConfig); + } database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index 2d59cf636e..621bcc676b 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -14,6 +14,10 @@ * limitations under the License. */ +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { Knex } from 'knex'; @@ -26,7 +30,14 @@ export interface DatabaseConnector { /** * createClient provides an instance of a knex database connector. */ - createClient(dbConfig: Config, overrides?: Partial): Knex; + createClient( + dbConfig: Config, + overrides?: Partial, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Knex; /** * createNameOverride provides a partial knex config sufficient to override a * database name. diff --git a/packages/backend-dev-utils/.eslintrc.js b/packages/backend-dev-utils/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/packages/backend-dev-utils/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/backend-dev-utils/README.md b/packages/backend-dev-utils/README.md new file mode 100644 index 0000000000..8acce94cf0 --- /dev/null +++ b/packages/backend-dev-utils/README.md @@ -0,0 +1,5 @@ +# backend-dev-utils + +Welcome to the backend-dev-utils package! + +_This package is experimental._ diff --git a/packages/backend-dev-utils/api-report.md b/packages/backend-dev-utils/api-report.md new file mode 100644 index 0000000000..216311ccd8 --- /dev/null +++ b/packages/backend-dev-utils/api-report.md @@ -0,0 +1,20 @@ +## API Report File for "@backstage/backend-dev-utils" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +// @public +export class DevDataStore { + static get(): DevDataStore | undefined; + load(key: string): Promise<{ + loaded: boolean; + data: T; + }>; + save( + key: string, + data: T, + ): Promise<{ + saved: boolean; + }>; +} +``` diff --git a/packages/backend-dev-utils/package.json b/packages/backend-dev-utils/package.json new file mode 100644 index 0000000000..735185e37f --- /dev/null +++ b/packages/backend-dev-utils/package.json @@ -0,0 +1,36 @@ +{ + "name": "@backstage/backend-dev-utils", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "node-library" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/backend-dev-utils" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "devDependencies": { + "@backstage/cli": "workspace:^" + }, + "files": [ + "dist" + ] +} diff --git a/packages/backend-dev-utils/src/DevDataStore.test.ts b/packages/backend-dev-utils/src/DevDataStore.test.ts new file mode 100644 index 0000000000..2bb3f221ae --- /dev/null +++ b/packages/backend-dev-utils/src/DevDataStore.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Serializable, spawn } from 'child_process'; +import { DevDataStore } from './DevDataStore'; +import { BackstageIpcClient } from './ipcClient'; + +function applyIpcTransform(value: Serializable): Promise { + const child = spawn( + 'node', + [ + '--eval', + ` + const listener = msg => { + process.send(msg); + process.removeListener('message', listener); + } + process.addListener('message', listener); + `, + ], + { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'], + serialization: 'advanced', + }, + ); + + return new Promise(resolve => { + child.addListener('message', resolve); + child.send(value); + }); +} + +class MockIpcClient implements Pick { + #data = new Map(); + + async request(method: string, body: any): Promise { + if (method === 'DevDataStore.save') { + this.#data.set(body.key, body.data); + return { saved: true }; + } else if (method === 'DevDataStore.load') { + if (!this.#data.has(body.key)) { + return { loaded: false, data: undefined }; + } + const data = this.#data.get(body.key)!; + return { loaded: true, data: await applyIpcTransform(data) }; + } + throw new Error('Unknown message'); + } +} + +describe('DevDataStore', () => { + it('should save and load data', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect(store.save('test', { foo: 'bar' })).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: { foo: 'bar' }, + }); + }); + + it('should save and load buffers', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', Buffer.from('abc', 'ascii')), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: Buffer.from('abc', 'ascii'), + }); + }); + + it('should save and load array buffers', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', new Uint8Array(Buffer.from('abc', 'ascii'))), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: new Uint8Array(Buffer.from('abc', 'ascii')), + }); + }); + + it('should save and load buffers nested in objects', async () => { + const store = DevDataStore.forTest(new MockIpcClient()); + + await expect( + store.save('test', { + x: Buffer.from('x', 'ascii'), + y: [Buffer.from('y', 'ascii')], + }), + ).resolves.toEqual({ + saved: true, + }); + await expect(store.load('test')).resolves.toEqual({ + loaded: true, + data: { x: Buffer.from('x', 'ascii'), y: [Buffer.from('y', 'ascii')] }, + }); + }); +}); diff --git a/packages/backend-dev-utils/src/DevDataStore.ts b/packages/backend-dev-utils/src/DevDataStore.ts new file mode 100644 index 0000000000..4069ab1bdb --- /dev/null +++ b/packages/backend-dev-utils/src/DevDataStore.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BackstageIpcClient, ipcClient } from './ipcClient'; + +interface SaveRequest { + key: string; + data: unknown; +} + +interface SaveResponse { + saved: boolean; +} + +interface LoadRequest { + key: string; +} + +interface LoadResponse { + loaded: boolean; + data: unknown; +} + +/** + * A data store that can be used to store temporary data during development. + * + * @public + */ +export class DevDataStore { + static #instance?: DevDataStore; + + /** + * Tries to acquire a DevDataStore instance. This will only succeed when the backend + * process is being run through the `@backstage/cli` in development mode. + * + * @returns A DevDataStore instance, or undefined if not available. + */ + static get(): DevDataStore | undefined { + if (ipcClient) { + if (!this.#instance) { + this.#instance = new DevDataStore(ipcClient); + } + return this.#instance; + } + return undefined; + } + + /** @internal */ + static forTest(client: Pick): DevDataStore { + return new DevDataStore(client as BackstageIpcClient); + } + + #client: BackstageIpcClient; + + private constructor(client: BackstageIpcClient) { + this.#client = client; + } + + /** + * Save data to the data store. + * + * @param key - The key used to identify the data. + * @param data - The data to save. The data will be serialized using advanced IPC serialization. + * @returns A promise that resolves to a result object that indicates whether the data was saved. + */ + async save(key: string, data: T): Promise<{ saved: boolean }> { + return this.#client.request( + 'DevDataStore.save', + { key, data }, + ); + } + + /** + * Loads data from the data store. + * + * @param key - The key used to identify the data. + * @returns A promise that resolves to a result object that indicates whether the data was loaded, as well as the data. + */ + async load(key: string): Promise<{ loaded: boolean; data: T }> { + const result = await this.#client.request( + 'DevDataStore.load', + { key }, + ); + return result as { loaded: boolean; data: T }; + } +} diff --git a/packages/backend-dev-utils/src/index.ts b/packages/backend-dev-utils/src/index.ts new file mode 100644 index 0000000000..9b707db029 --- /dev/null +++ b/packages/backend-dev-utils/src/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Local development helpers for backend. + * + * @packageDocumentation + */ + +export { DevDataStore } from './DevDataStore'; diff --git a/packages/backend-dev-utils/src/ipcClient.ts b/packages/backend-dev-utils/src/ipcClient.ts new file mode 100644 index 0000000000..87c1596a39 --- /dev/null +++ b/packages/backend-dev-utils/src/ipcClient.ts @@ -0,0 +1,123 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type SendMessage = Exclude; + +interface Request { + id: number; + method: string; + body: unknown; + type: string; +} + +type Response = + | { + type: string; + id: number; + body: unknown; + } + | { + type: string; + id: number; + error: Error; + }; + +const requestType = '@backstage/cli/channel/request'; +const responseType = '@backstage/cli/channel/response'; + +/** + * The client side of an IPC communication channel. + * + * @internal + */ +export class BackstageIpcClient { + #messageId = 0; + #sendMessage: SendMessage; + + /** + * Creates a new client if we're in a child process with IPC and BACKSTAGE_CLI_CHANNEL is set. + */ + static create(): BackstageIpcClient | undefined { + const sendMessage = process.send?.bind(process); + return sendMessage && process.env.BACKSTAGE_CLI_CHANNEL + ? new BackstageIpcClient(sendMessage) + : undefined; + } + + constructor(sendMessage: SendMessage) { + this.#sendMessage = sendMessage; + } + + /** + * Send a request to the parent process and wait for a response. + */ + async request( + method: string, + body: TRequestBody, + ): Promise { + return new Promise((resolve, reject) => { + const id = this.#messageId++; + + const request: Request = { + type: requestType, + id, + method, + body, + }; + + this.#sendMessage(request, (e: Error) => { + if (e) { + reject(e); + return; + } + + let timeout: NodeJS.Timeout | undefined = undefined; + + const messageHandler = (response: Response) => { + if (response?.type !== responseType) { + return; + } + if (response.id !== id) { + return; + } + + if ('error' in response) { + const error = new Error(response.error.message); + if (response.error.name) { + error.name = response.error.name; + } + reject(error); + } else { + resolve(response.body as TResponseBody); + } + + clearTimeout(timeout); + process.removeListener('message', messageHandler); + }; + + timeout = setTimeout(() => { + reject(new Error(`IPC request '${method}' with ID ${id} timed out`)); + process.removeListener('message', messageHandler); + }, 5000); + timeout.unref(); + + process.addListener('message', messageHandler as () => void); + }); + }); + } +} + +export const ipcClient = BackstageIpcClient.create(); diff --git a/packages/backend-dev-utils/src/setupTests.ts b/packages/backend-dev-utils/src/setupTests.ts new file mode 100644 index 0000000000..4b9026cde5 --- /dev/null +++ b/packages/backend-dev-utils/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/packages/backend-next/package.json b/packages/backend-next/package.json index cc537ef514..fb821e19eb 100644 --- a/packages/backend-next/package.json +++ b/packages/backend-next/package.json @@ -18,7 +18,7 @@ "backstage" ], "scripts": { - "start": "backstage-cli package start", + "start": "EXPERIMENTAL_BACKEND_START=1 backstage-cli package start", "build": "backstage-cli package build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", diff --git a/packages/cli/package.json b/packages/cli/package.json index 909ac8460c..e6b9b7a97e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -36,6 +36,7 @@ "@backstage/errors": "workspace:^", "@backstage/release-manifests": "workspace:^", "@backstage/types": "workspace:^", + "@esbuild-kit/cjs-loader": "^2.4.1", "@manypkg/get-packages": "^1.1.3", "@octokit/request": "^6.0.0", "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7", @@ -65,6 +66,7 @@ "chalk": "^4.0.0", "chokidar": "^3.3.1", "commander": "^9.1.0", + "cross-spawn": "^7.0.3", "css-loader": "^6.5.1", "diff": "^5.0.0", "esbuild": "^0.17.0", @@ -136,6 +138,7 @@ "@backstage/dev-utils": "workspace:^", "@backstage/test-utils": "workspace:^", "@backstage/theme": "workspace:^", + "@types/cross-spawn": "^6.0.2", "@types/diff": "^5.0.0", "@types/express": "^4.17.6", "@types/fs-extra": "^9.0.1", diff --git a/packages/cli/src/commands/start/startBackend.ts b/packages/cli/src/commands/start/startBackend.ts index 61ada7a291..3e6a13be96 100644 --- a/packages/cli/src/commands/start/startBackend.ts +++ b/packages/cli/src/commands/start/startBackend.ts @@ -17,6 +17,7 @@ import fs from 'fs-extra'; import { paths } from '../../lib/paths'; import { serveBackend } from '../../lib/bundler'; +import { startBackendExperimental } from '../../lib/experimental/startBackendExperimental'; interface StartBackendOptions { checksEnabled: boolean; @@ -25,17 +26,28 @@ interface StartBackendOptions { } export async function startBackend(options: StartBackendOptions) { - // Cleaning dist/ before we start the dev process helps work around an issue - // where we end up with the entrypoint executing multiple times, causing - // a port bind conflict among other things. - await fs.remove(paths.resolveTarget('dist')); + if (process.env.EXPERIMENTAL_BACKEND_START) { + const waitForExit = await startBackendExperimental({ + entry: 'src/index', + checksEnabled: false, // not supported + inspectEnabled: options.inspectEnabled, + inspectBrkEnabled: options.inspectBrkEnabled, + }); - const waitForExit = await serveBackend({ - entry: 'src/index', - checksEnabled: options.checksEnabled, - inspectEnabled: options.inspectEnabled, - inspectBrkEnabled: options.inspectBrkEnabled, - }); + await waitForExit(); + } else { + // Cleaning dist/ before we start the dev process helps work around an issue + // where we end up with the entrypoint executing multiple times, causing + // a port bind conflict among other things. + await fs.remove(paths.resolveTarget('dist')); - await waitForExit(); + const waitForExit = await serveBackend({ + entry: 'src/index', + checksEnabled: options.checksEnabled, + inspectEnabled: options.inspectEnabled, + inspectBrkEnabled: options.inspectBrkEnabled, + }); + + await waitForExit(); + } } diff --git a/packages/cli/src/lib/experimental/IpcServer.ts b/packages/cli/src/lib/experimental/IpcServer.ts new file mode 100644 index 0000000000..8ceb72b046 --- /dev/null +++ b/packages/cli/src/lib/experimental/IpcServer.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeError } from '@backstage/errors'; +import { ChildProcess } from 'child_process'; + +interface RequestMeta { + generation: number; +} + +type MethodHandler = ( + req: TRequest, + meta: RequestMeta, +) => Promise; + +interface Request { + id: number; + method: string; + body: unknown; + type: string; +} + +const requestType = '@backstage/cli/channel/request'; +const responseType = '@backstage/cli/channel/response'; + +export class IpcServer { + #generation = 1; + #methods = new Map>(); + + addChild(child: ChildProcess) { + const generation = this.#generation++; + const sendMessage = child.send?.bind(child); + if (!sendMessage) { + return; + } + + const messageListener = (request: Request) => { + if (request.type !== requestType) { + return; + } + + const handler = this.#methods.get(request.method); + if (!handler) { + sendMessage({ + type: responseType, + id: request.id, + error: { + name: 'NotFoundError', + message: `No handler registered for method ${request.method}`, + }, + }); + return; + } + + Promise.resolve() + .then(() => handler(request.body, { generation })) + .then(response => + sendMessage({ + type: responseType, + id: request.id, + body: response, + }), + ) + .catch(error => + sendMessage({ + type: responseType, + id: request.id, + error: serializeError(error), + }), + ); + }; + + child.addListener('message', messageListener as (req: unknown) => void); + + child.addListener('exit', () => { + child.removeListener('message', messageListener); + }); + } + + registerMethod( + method: string, + handler: MethodHandler, + ) { + if (this.#methods.has(method)) { + throw new Error(`A handler is already registered for method ${method}`); + } + this.#methods.set(method, handler); + } +} diff --git a/packages/cli/src/lib/experimental/ServerDataStore.ts b/packages/cli/src/lib/experimental/ServerDataStore.ts new file mode 100644 index 0000000000..c0232157cb --- /dev/null +++ b/packages/cli/src/lib/experimental/ServerDataStore.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IpcServer } from './IpcServer'; + +interface StorageItem { + generation: number; + data: unknown; +} + +interface SaveRequest { + key: string; + data: unknown; +} + +interface SaveResponse { + saved: boolean; +} + +interface LoadRequest { + key: string; +} + +interface LoadResponse { + loaded: boolean; + data: unknown; +} + +export class ServerDataStore { + static bind(server: IpcServer): void { + const store = new Map(); + + server.registerMethod( + 'DevDataStore.save', + async (request, { generation }) => { + const { key, data } = request; + if (!key) { + throw new Error('Key is required in DevDataStore.save'); + } + + const item = store.get(key); + + if (!item) { + store.set(key, { generation, data }); + return { saved: true }; + } + + if (item.generation > generation) { + return { saved: false }; + } + + store.set(key, { generation, data }); + return { saved: true }; + }, + ); + + server.registerMethod( + 'DevDataStore.load', + async request => { + const item = store.get(request.key); + return { loaded: Boolean(item), data: item?.data }; + }, + ); + } +} diff --git a/packages/cli/src/lib/experimental/startBackendExperimental.ts b/packages/cli/src/lib/experimental/startBackendExperimental.ts new file mode 100644 index 0000000000..c08fbda0da --- /dev/null +++ b/packages/cli/src/lib/experimental/startBackendExperimental.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BackendServeOptions } from '../bundler/types'; +import type { ChildProcess } from 'child_process'; +import { fileURLToPath } from 'url'; +import { isAbsolute as isAbsolutePath } from 'path'; +import { FSWatcher, watch } from 'chokidar'; +import { IpcServer } from './IpcServer'; +import { ServerDataStore } from './ServerDataStore'; +import debounce from 'lodash/debounce'; +import spawn from 'cross-spawn'; +import { paths } from '../paths'; + +const loaderArgs = ['--require', require.resolve('@esbuild-kit/cjs-loader')]; + +export async function startBackendExperimental(options: BackendServeOptions) { + const envEnv = process.env as { NODE_ENV: string }; + if (!envEnv.NODE_ENV) { + envEnv.NODE_ENV = 'development'; + } + + // Set up the parent IPC server and bind the available services + const server = new IpcServer(); + ServerDataStore.bind(server); + + let exiting = false; + let child: ChildProcess | undefined; + let watcher: FSWatcher | undefined = undefined; + let shutdownPromise: Promise | undefined = undefined; + + const restart = debounce(async () => { + // If a re-trigger happens during an existing shutdown, we just ignore it + if (shutdownPromise) { + return; + } + + if (child && !child.killed && child.exitCode === null) { + // We always wait for the existing process to exit, to make sure we don't get IPC conflicts + shutdownPromise = new Promise(resolve => child!.once('exit', resolve)); + child.kill(); + await shutdownPromise; + shutdownPromise = undefined; + } + + // We've received a shutdown signal + if (exiting) { + return; + } + + const optionArgs = new Array(); + if (options.inspectEnabled) { + optionArgs.push('--inspect'); + } else if (options.inspectBrkEnabled) { + optionArgs.push('--inspect-brk'); + } + + const userArgs = process.argv + .slice(['node', 'backstage-cli', 'package', 'start'].length) + .filter(arg => !optionArgs.includes(arg)); + + child = spawn( + process.execPath, + [...loaderArgs, ...optionArgs, options.entry, ...userArgs], + { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'], + env: { + ...process.env, + BACKSTAGE_CLI_CHANNEL: '1', + ESBK_TSCONFIG_PATH: paths.resolveTargetRoot('tsconfig.json'), + }, + serialization: 'advanced', + }, + ); + + server.addChild(child); + + // This captures messages sent by @esbuild-kit/cjs-loader + child.on('message', (data: { type?: string } | null) => { + if (typeof data === 'object' && data?.type === 'dependency') { + let path = (data as { path: string }).path; + if (path.startsWith('file:')) { + path = fileURLToPath(path); + } + + if (isAbsolutePath(path)) { + watcher?.add(path); + } + } + }); + }, 100); + + restart(); + + watcher = watch([paths.targetDir], { + cwd: process.cwd(), + ignored: ['**/.*/**', '**/node_modules/**'], + ignoreInitial: true, + ignorePermissionErrors: true, + }).on('all', restart); + + // Trigger restart on hitting enter in the terminal + process.stdin.on('data', restart); + + const exitPromise = new Promise(resolveExitPromise => { + async function handleSignal(signal: NodeJS.Signals) { + exiting = true; + + // Forward signals to child and wait for it to exit if still running + if (child && child.exitCode === null) { + await new Promise(resolve => { + child!.on('close', resolve); + child!.kill(signal); + }); + } + + resolveExitPromise(); + } + + process.once('SIGINT', handleSignal); + process.once('SIGTERM', handleSignal); + }); + + return () => exitPromise; +} diff --git a/packages/cli/src/types.d.ts b/packages/cli/src/types.d.ts index fedbe61f02..1222b2b4b8 100644 --- a/packages/cli/src/types.d.ts +++ b/packages/cli/src/types.d.ts @@ -258,3 +258,5 @@ declare module 'webpack-node-externals' { } } } + +declare module '@esbuild-kit/cjs-loader' {} diff --git a/yarn.lock b/yarn.lock index f042b70bf5..97b968a868 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3423,6 +3423,7 @@ __metadata: resolution: "@backstage/backend-common@workspace:packages/backend-common" dependencies: "@backstage/backend-app-api": "workspace:^" + "@backstage/backend-dev-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" @@ -3519,6 +3520,14 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-dev-utils@workspace:^, @backstage/backend-dev-utils@workspace:packages/backend-dev-utils": + version: 0.0.0-use.local + resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils" + dependencies: + "@backstage/cli": "workspace:^" + languageName: unknown + linkType: soft + "@backstage/backend-plugin-api@workspace:^, @backstage/backend-plugin-api@workspace:packages/backend-plugin-api": version: 0.0.0-use.local resolution: "@backstage/backend-plugin-api@workspace:packages/backend-plugin-api" @@ -3664,6 +3673,7 @@ __metadata: "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" "@backstage/types": "workspace:^" + "@esbuild-kit/cjs-loader": ^2.4.1 "@manypkg/get-packages": ^1.1.3 "@octokit/request": ^6.0.0 "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 @@ -3682,6 +3692,7 @@ __metadata: "@swc/core": ^1.3.9 "@swc/helpers": ^0.4.7 "@swc/jest": ^0.2.22 + "@types/cross-spawn": ^6.0.2 "@types/diff": ^5.0.0 "@types/express": ^4.17.6 "@types/fs-extra": ^9.0.1 @@ -3709,6 +3720,7 @@ __metadata: chalk: ^4.0.0 chokidar: ^3.3.1 commander: ^9.1.0 + cross-spawn: ^7.0.3 css-loader: ^6.5.1 del: ^6.0.0 diff: ^5.0.0 @@ -9188,6 +9200,26 @@ __metadata: languageName: node linkType: hard +"@esbuild-kit/cjs-loader@npm:^2.4.1": + version: 2.4.1 + resolution: "@esbuild-kit/cjs-loader@npm:2.4.1" + dependencies: + "@esbuild-kit/core-utils": ^3.0.0 + get-tsconfig: ^4.2.0 + checksum: a516065907be0ead76ac2199ccb08ff92659ba5e2edb4bb8772b6a63afe4faed7eb45a3b4d87266a68c7c135c3dba971cd087bc6f16c382356e835c7dd3440f5 + languageName: node + linkType: hard + +"@esbuild-kit/core-utils@npm:^3.0.0": + version: 3.0.0 + resolution: "@esbuild-kit/core-utils@npm:3.0.0" + dependencies: + esbuild: ~0.15.10 + source-map-support: ^0.5.21 + checksum: 0e89ec718e2211bf95c48a8085aaef88e8e416f42abd1c62d488d5458eecd3fbc144179a0c5570ad36fa7e2d3bbc411f8d3fb28802c37ced2154dc2c6ded9dfe + languageName: node + linkType: hard + "@esbuild/android-arm64@npm:0.17.5": version: 0.17.5 resolution: "@esbuild/android-arm64@npm:0.17.5" @@ -13942,6 +13974,15 @@ __metadata: languageName: node linkType: hard +"@types/cross-spawn@npm:^6.0.2": + version: 6.0.2 + resolution: "@types/cross-spawn@npm:6.0.2" + dependencies: + "@types/node": "*" + checksum: fa9edd32178878cab3ea8d6d0260639e0fe4860ddb3887b8de53d6e8036e154fc5f313c653f690975aa25025aea8beb83fb0870b931bf8d9202c3ac530a24c9d + languageName: node + linkType: hard + "@types/d3-color@npm:*": version: 3.0.2 resolution: "@types/d3-color@npm:3.0.2" @@ -21365,7 +21406,7 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.15.6": +"esbuild@npm:^0.15.6, esbuild@npm:~0.15.10": version: 0.15.18 resolution: "esbuild@npm:0.15.18" dependencies: @@ -23523,6 +23564,13 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.2.0": + version: 4.3.0 + resolution: "get-tsconfig@npm:4.3.0" + checksum: 2597aab99aa3a24db209e192a3e5874ac47fc5abc71703ee26346e0c5816cb346ca09fc813c739db5862d3a2905d89aeca1b0cbc46c2b272398d672309aaf414 + languageName: node + linkType: hard + "getopts@npm:2.3.0": version: 2.3.0 resolution: "getopts@npm:2.3.0" @@ -35176,7 +35224,7 @@ __metadata: languageName: node linkType: hard -"source-map-support@npm:^0.5.16, source-map-support@npm:~0.5.20": +"source-map-support@npm:^0.5.16, source-map-support@npm:^0.5.21, source-map-support@npm:~0.5.20": version: 0.5.21 resolution: "source-map-support@npm:0.5.21" dependencies: