Merge pull request #16103 from backstage/mob/bs-entry

backend-system: add new experimental dev setup

.changeset/green-adults-retire.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-dev-utils': minor
+---
+
+Introduced a new package for backend development utilities. Similar to how `@backstage/dev-utils` is used in the frontend.

.changeset/long-chefs-burn.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+Updated database factory to pass service deps required for restoring database state during development.

.changeset/old-ways-know.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+The `DatabaseManager.forPlugin` method now accepts additional service dependencies. There is no need to update existing code to pass these dependencies.

.changeset/red-kiwis-compare.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+The shutdown signal handlers are now installed as part of the backend instance rather than the lifecycle service, and explicitly cause the process to exit.

.changeset/weak-radios-sing.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Added an experimental mode for the `package start` command for backend packages. Enabled by setting `EXPERIMENTAL_BACKEND_START`.

packages/backend-app-api/src/services/implementations/database/databaseFactory.ts

@@ -26,7 +26,8 @@ export const databaseFactory = createServiceFactory({
   service: coreServices.database,
   deps: {
     config: coreServices.config,
-    plugin: coreServices.pluginMetadata,
+    lifecycle: coreServices.lifecycle,
+    pluginMetadata: coreServices.pluginMetadata,
   },
   async createRootContext({ config }) {
     return config.getOptional('backend.database')
@@ -39,7 +40,10 @@ export const databaseFactory = createServiceFactory({
           }),
         );
   },
-  async factory({ plugin }, databaseManager) {
-    return databaseManager.forPlugin(plugin.getId());
+  async factory({ pluginMetadata, lifecycle }, databaseManager) {
+    return databaseManager.forPlugin(pluginMetadata.getId(), {
+      pluginMetadata,
+      lifecycle,
+    });
   },
 });

packages/backend-app-api/src/services/implementations/rootLifecycle/rootLifecycleFactory.ts

@@ -22,11 +22,8 @@ import {
   LoggerService,
 } from '@backstage/backend-plugin-api';
 
-const CALLBACKS = ['SIGTERM', 'SIGINT', 'beforeExit'];
 export class BackendLifecycleImpl implements RootLifecycleService {
-  constructor(private readonly logger: LoggerService) {
-    CALLBACKS.map(signal => process.on(signal, () => this.shutdown()));
-  }
+  constructor(private readonly logger: LoggerService) {}
 
   #isCalled = false;
   #shutdownTasks: Array<LifecycleServiceShutdownHook> = [];

packages/backend-app-api/src/wiring/BackendInitializer.ts

@@ -28,7 +28,7 @@ import {
 } from './types';
 
 export class BackendInitializer {
-  #started = false;
+  #startPromise?: Promise<void>;
   #features = new Map<BackendFeature, unknown>();
   #registerInits = new Array<BackendRegisterInit>();
   #extensionPoints = new Map<ExtensionPoint<unknown>, unknown>();
@@ -75,18 +75,40 @@ export class BackendInitializer {
   }
 
   add<TOptions>(feature: BackendFeature, options?: TOptions) {
-    if (this.#started) {
+    if (this.#startPromise) {
       throw new Error('feature can not be added after the backend has started');
     }
     this.#features.set(feature, options);
   }
 
   async start(): Promise<void> {
-    if (this.#started) {
+    if (this.#startPromise) {
       throw new Error('Backend has already started');
     }
-    this.#started = true;
 
+    const exitHandler = async () => {
+      process.removeListener('SIGTERM', exitHandler);
+      process.removeListener('SIGINT', exitHandler);
+      process.removeListener('beforeExit', exitHandler);
+
+      try {
+        await this.stop();
+        process.exit(0);
+      } catch (error) {
+        console.error(error);
+        process.exit(1);
+      }
+    };
+
+    process.addListener('SIGTERM', exitHandler);
+    process.addListener('SIGINT', exitHandler);
+    process.addListener('beforeExit', exitHandler);
+
+    this.#startPromise = this.#doStart();
+    await this.#startPromise;
+  }
+
+  async #doStart(): Promise<void> {
     // Initialize all root scoped services
     for (const ref of this.#serviceHolder.getServiceRefs()) {
       if (ref.scope === 'root') {
@@ -177,9 +199,10 @@ export class BackendInitializer {
   }
 
   async stop(): Promise<void> {
-    if (!this.#started) {
+    if (!this.#startPromise) {
       return;
     }
+    await this.#startPromise;
 
     const lifecycleService = await this.#serviceHolder.get(
       coreServices.rootLifecycle,

packages/backend-common/api-report.md

@@ -32,6 +32,7 @@ import { IdentityService } from '@backstage/backend-plugin-api';
 import { isChildPath } from '@backstage/cli-common';
 import { Knex } from 'knex';
 import { KubeConfig } from '@kubernetes/client-node';
+import { LifecycleService } from '@backstage/backend-plugin-api';
 import { LoadConfigOptionsRemote } from '@backstage/config-loader';
 import { Logger } from 'winston';
 import { LoggerService } from '@backstage/backend-plugin-api';
@@ -40,6 +41,7 @@ import { PermissionsService } from '@backstage/backend-plugin-api';
 import { CacheService as PluginCacheManager } from '@backstage/backend-plugin-api';
 import { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api';
 import { DiscoveryService as PluginEndpointDiscovery } from '@backstage/backend-plugin-api';
+import { PluginMetadataService } from '@backstage/backend-plugin-api';
 import { PushResult } from 'isomorphic-git';
 import { Readable } from 'stream';
 import { ReadCommitResult } from 'isomorphic-git';
@@ -232,6 +234,10 @@ export class Contexts {
 export function createDatabaseClient(
   dbConfig: Config,
   overrides?: Partial<Knex.Config>,
+  deps?: {
+    lifecycle: LifecycleService;
+    pluginMetadata: PluginMetadataService;
+  },
 ): Knex<any, any[]>;
 
 // @public
@@ -252,7 +258,13 @@ export function createStatusCheckRouter(options: {
 
 // @public
 export class DatabaseManager {
-  forPlugin(pluginId: string): PluginDatabaseManager;
+  forPlugin(
+    pluginId: string,
+    deps?: {
+      lifecycle: LifecycleService;
+      pluginMetadata: PluginMetadataService;
+    },
+  ): PluginDatabaseManager;
   static fromConfig(
     config: Config,
     options?: DatabaseManagerOptions,

packages/backend-common/package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "@backstage/backend-app-api": "workspace:^",
+    "@backstage/backend-dev-utils": "workspace:^",
     "@backstage/backend-plugin-api": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/config": "workspace:^",

packages/backend-common/src/database/DatabaseManager.ts

@@ -29,7 +29,11 @@ import {
 } from './connection';
 import { PluginDatabaseManager } from './types';
 import path from 'path';
-import { LoggerService } from '@backstage/backend-plugin-api';
+import {
+  LifecycleService,
+  LoggerService,
+  PluginMetadataService,
+} from '@backstage/backend-plugin-api';
 import { stringifyError } from '@backstage/errors';
 
 /**
@@ -94,12 +98,18 @@ export class DatabaseManager {
    * should be unique as they are used to look up database config overrides under
    * `backend.database.plugin`.
    */
-  forPlugin(pluginId: string): PluginDatabaseManager {
+  forPlugin(
+    pluginId: string,
+    deps?: {
+      lifecycle: LifecycleService;
+      pluginMetadata: PluginMetadataService;
+    },
+  ): PluginDatabaseManager {
     const _this = this;
 
     return {
       getClient(): Promise<Knex> {
-        return _this.getDatabase(pluginId);
+        return _this.getDatabase(pluginId, deps);
       },
       migrations: {
         skip: false,
@@ -307,7 +317,13 @@ export class DatabaseManager {
    * @returns Promise which resolves to a scoped Knex database client for a
    *          plugin
    */
-  private async getDatabase(pluginId: string): Promise<Knex> {
+  private async getDatabase(
+    pluginId: string,
+    deps?: {
+      lifecycle: LifecycleService;
+      pluginMetadata: PluginMetadataService;
+    },
+  ): Promise<Knex> {
     if (this.databaseCache.has(pluginId)) {
       return this.databaseCache.get(pluginId)!;
     }
@@ -351,6 +367,7 @@ export class DatabaseManager {
       const client = createDatabaseClient(
         pluginConfig,
         databaseClientOverrides,
+        deps,
       );
       this.startKeepaliveLoop(pluginId, client);
       return client;

packages/backend-common/src/database/connection.ts

@@ -22,6 +22,10 @@ import { mergeDatabaseConfig } from './config';
 import { DatabaseConnector } from './types';
 
 import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors';
+import {
+  LifecycleService,
+  PluginMetadataService,
+} from '@backstage/backend-plugin-api';
 
 type DatabaseClient =
   | 'pg'
@@ -55,11 +59,15 @@ const ConnectorMapping: Record<DatabaseClient, DatabaseConnector> = {
 export function createDatabaseClient(
   dbConfig: Config,
   overrides?: Partial<Knex.Config>,
+  deps?: {
+    lifecycle: LifecycleService;
+    pluginMetadata: PluginMetadataService;
+  },
 ) {
   const client: DatabaseClient = dbConfig.getString('client');
 
   return (
-    ConnectorMapping[client]?.createClient(dbConfig, overrides) ??
+    ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ??
     knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides))
   );
 }

packages/backend-common/src/database/connectors/sqlite3.ts

@@ -18,8 +18,13 @@ import { Config } from '@backstage/config';
 import { ensureDirSync } from 'fs-extra';
 import knexFactory, { Knex } from 'knex';
 import path from 'path';
+import { DevDataStore } from '@backstage/backend-dev-utils';
 import { mergeDatabaseConfig } from '../config';
 import { DatabaseConnector } from '../types';
+import {
+  LifecycleService,
+  PluginMetadataService,
+} from '@backstage/backend-plugin-api';
 
 /**
  * Creates a knex SQLite3 database connection
@@ -30,22 +35,65 @@ import { DatabaseConnector } from '../types';
 export function createSqliteDatabaseClient(
   dbConfig: Config,
   overrides?: Knex.Config,
+  deps?: {
+    lifecycle: LifecycleService;
+    pluginMetadata: PluginMetadataService;
+  },
 ) {
   const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides);
+  const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
+
+  const filename = connConfig.filename ?? ':memory:';
 
   // If storage on disk is used, ensure that the directory exists
-  if (
-    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename &&
-    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !==
-      ':memory:'
-  ) {
-    const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
+  if (filename !== ':memory:') {
     const directory = path.dirname(filename);
-
     ensureDirSync(directory);
   }
 
-  const database = knexFactory(knexConfig);
+  let database: Knex;
+
+  if (deps && filename === ':memory:') {
+    // The dev store is used during watch mode to store and restore the database
+    // across reloads. It is only available when running the backend through
+    // `backstage-cli package start`.
+    const devStore = DevDataStore.get();
+
+    if (devStore) {
+      const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`;
+
+      const connectionLoader = async () => {
+        // If seed data is available, use it tconnectionLoader restore the database
+        const { data: seedData } = await devStore.load(dataKey);
+
+        return {
+          ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig),
+          filename: seedData ?? ':memory:',
+        };
+      };
+
+      database = knexFactory({
+        ...knexConfig,
+        connection: Object.assign(connectionLoader, {
+          // This is a workaround for the knex SQLite driver always warning when using a config loader
+          filename: ':memory:',
+        }),
+      });
+
+      // If the dev store is available we save the database state on shutdown
+      deps.lifecycle.addShutdownHook({
+        async fn() {
+          const connection = await database.client.acquireConnection();
+          const data = connection.serialize();
+          await devStore.save(dataKey, data);
+        },
+      });
+    } else {
+      database = knexFactory(knexConfig);
+    }
+  } else {
+    database = knexFactory(knexConfig);
+  }
 
   database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
     resource.run('PRAGMA foreign_keys = ON', () => {});

packages/backend-common/src/database/types.ts

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+import {
+  LifecycleService,
+  PluginMetadataService,
+} from '@backstage/backend-plugin-api';
 import { Config } from '@backstage/config';
 import { Knex } from 'knex';
 
@@ -26,7 +30,14 @@ export interface DatabaseConnector {
   /**
    * createClient provides an instance of a knex database connector.
    */
-  createClient(dbConfig: Config, overrides?: Partial<Knex.Config>): Knex;
+  createClient(
+    dbConfig: Config,
+    overrides?: Partial<Knex.Config>,
+    deps?: {
+      lifecycle: LifecycleService;
+      pluginMetadata: PluginMetadataService;
+    },
+  ): Knex;
   /**
    * createNameOverride provides a partial knex config sufficient to override a
    * database name.

packages/backend-dev-utils/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

packages/backend-dev-utils/README.md

@@ -0,0 +1,5 @@
+# backend-dev-utils
+
+Welcome to the backend-dev-utils package!
+
+_This package is experimental._

packages/backend-dev-utils/api-report.md

@@ -0,0 +1,20 @@
+## API Report File for "@backstage/backend-dev-utils"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+// @public
+export class DevDataStore {
+  static get(): DevDataStore | undefined;
+  load<T>(key: string): Promise<{
+    loaded: boolean;
+    data: T;
+  }>;
+  save<T>(
+    key: string,
+    data: T,
+  ): Promise<{
+    saved: boolean;
+  }>;
+}
+```

packages/backend-dev-utils/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@backstage/backend-dev-utils",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "node-library"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "packages/backend-dev-utils"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "devDependencies": {
+    "@backstage/cli": "workspace:^"
+  },
+  "files": [
+    "dist"
+  ]
+}

packages/backend-dev-utils/src/DevDataStore.test.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Serializable, spawn } from 'child_process';
+import { DevDataStore } from './DevDataStore';
+import { BackstageIpcClient } from './ipcClient';
+
+function applyIpcTransform(value: Serializable): Promise<Serializable> {
+  const child = spawn(
+    'node',
+    [
+      '--eval',
+      `
+        const listener = msg => {
+          process.send(msg);
+          process.removeListener('message', listener);
+        }
+        process.addListener('message', listener);
+      `,
+    ],
+    {
+      stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+      serialization: 'advanced',
+    },
+  );
+
+  return new Promise(resolve => {
+    child.addListener('message', resolve);
+    child.send(value);
+  });
+}
+
+class MockIpcClient implements Pick<BackstageIpcClient, 'request'> {
+  #data = new Map<string, Serializable>();
+
+  async request(method: string, body: any): Promise<any> {
+    if (method === 'DevDataStore.save') {
+      this.#data.set(body.key, body.data);
+      return { saved: true };
+    } else if (method === 'DevDataStore.load') {
+      if (!this.#data.has(body.key)) {
+        return { loaded: false, data: undefined };
+      }
+      const data = this.#data.get(body.key)!;
+      return { loaded: true, data: await applyIpcTransform(data) };
+    }
+    throw new Error('Unknown message');
+  }
+}
+
+describe('DevDataStore', () => {
+  it('should save and load data', async () => {
+    const store = DevDataStore.forTest(new MockIpcClient());
+
+    await expect(store.save('test', { foo: 'bar' })).resolves.toEqual({
+      saved: true,
+    });
+    await expect(store.load('test')).resolves.toEqual({
+      loaded: true,
+      data: { foo: 'bar' },
+    });
+  });
+
+  it('should save and load buffers', async () => {
+    const store = DevDataStore.forTest(new MockIpcClient());
+
+    await expect(
+      store.save('test', Buffer.from('abc', 'ascii')),
+    ).resolves.toEqual({
+      saved: true,
+    });
+    await expect(store.load('test')).resolves.toEqual({
+      loaded: true,
+      data: Buffer.from('abc', 'ascii'),
+    });
+  });
+
+  it('should save and load array buffers', async () => {
+    const store = DevDataStore.forTest(new MockIpcClient());
+
+    await expect(
+      store.save('test', new Uint8Array(Buffer.from('abc', 'ascii'))),
+    ).resolves.toEqual({
+      saved: true,
+    });
+    await expect(store.load('test')).resolves.toEqual({
+      loaded: true,
+      data: new Uint8Array(Buffer.from('abc', 'ascii')),
+    });
+  });
+
+  it('should save and load buffers nested in objects', async () => {
+    const store = DevDataStore.forTest(new MockIpcClient());
+
+    await expect(
+      store.save('test', {
+        x: Buffer.from('x', 'ascii'),
+        y: [Buffer.from('y', 'ascii')],
+      }),
+    ).resolves.toEqual({
+      saved: true,
+    });
+    await expect(store.load('test')).resolves.toEqual({
+      loaded: true,
+      data: { x: Buffer.from('x', 'ascii'), y: [Buffer.from('y', 'ascii')] },
+    });
+  });
+});

packages/backend-dev-utils/src/DevDataStore.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { BackstageIpcClient, ipcClient } from './ipcClient';
+
+interface SaveRequest {
+  key: string;
+  data: unknown;
+}
+
+interface SaveResponse {
+  saved: boolean;
+}
+
+interface LoadRequest {
+  key: string;
+}
+
+interface LoadResponse {
+  loaded: boolean;
+  data: unknown;
+}
+
+/**
+ * A data store that can be used to store temporary data during development.
+ *
+ * @public
+ */
+export class DevDataStore {
+  static #instance?: DevDataStore;
+
+  /**
+   * Tries to acquire a DevDataStore instance. This will only succeed when the backend
+   * process is being run through the `@backstage/cli` in development mode.
+   *
+   * @returns A DevDataStore instance, or undefined if not available.
+   */
+  static get(): DevDataStore | undefined {
+    if (ipcClient) {
+      if (!this.#instance) {
+        this.#instance = new DevDataStore(ipcClient);
+      }
+      return this.#instance;
+    }
+    return undefined;
+  }
+
+  /** @internal */
+  static forTest(client: Pick<BackstageIpcClient, 'request'>): DevDataStore {
+    return new DevDataStore(client as BackstageIpcClient);
+  }
+
+  #client: BackstageIpcClient;
+
+  private constructor(client: BackstageIpcClient) {
+    this.#client = client;
+  }
+
+  /**
+   * Save data to the data store.
+   *
+   * @param key - The key used to identify the data.
+   * @param data - The data to save. The data will be serialized using advanced IPC serialization.
+   * @returns A promise that resolves to a result object that indicates whether the data was saved.
+   */
+  async save<T>(key: string, data: T): Promise<{ saved: boolean }> {
+    return this.#client.request<SaveRequest, SaveResponse>(
+      'DevDataStore.save',
+      { key, data },
+    );
+  }
+
+  /**
+   * Loads data from the data store.
+   *
+   * @param key - The key used to identify the data.
+   * @returns A promise that resolves to a result object that indicates whether the data was loaded, as well as the data.
+   */
+  async load<T>(key: string): Promise<{ loaded: boolean; data: T }> {
+    const result = await this.#client.request<LoadRequest, LoadResponse>(
+      'DevDataStore.load',
+      { key },
+    );
+    return result as { loaded: boolean; data: T };
+  }
+}

packages/backend-dev-utils/src/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Local development helpers for backend.
+ *
+ * @packageDocumentation
+ */
+
+export { DevDataStore } from './DevDataStore';

packages/backend-dev-utils/src/ipcClient.ts

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+type SendMessage = Exclude<typeof process.send, undefined>;
+
+interface Request {
+  id: number;
+  method: string;
+  body: unknown;
+  type: string;
+}
+
+type Response =
+  | {
+      type: string;
+      id: number;
+      body: unknown;
+    }
+  | {
+      type: string;
+      id: number;
+      error: Error;
+    };
+
+const requestType = '@backstage/cli/channel/request';
+const responseType = '@backstage/cli/channel/response';
+
+/**
+ * The client side of an IPC communication channel.
+ *
+ * @internal
+ */
+export class BackstageIpcClient {
+  #messageId = 0;
+  #sendMessage: SendMessage;
+
+  /**
+   * Creates a new client if we're in a child process with IPC and BACKSTAGE_CLI_CHANNEL is set.
+   */
+  static create(): BackstageIpcClient | undefined {
+    const sendMessage = process.send?.bind(process);
+    return sendMessage && process.env.BACKSTAGE_CLI_CHANNEL
+      ? new BackstageIpcClient(sendMessage)
+      : undefined;
+  }
+
+  constructor(sendMessage: SendMessage) {
+    this.#sendMessage = sendMessage;
+  }
+
+  /**
+   * Send a request to the parent process and wait for a response.
+   */
+  async request<TRequestBody, TResponseBody>(
+    method: string,
+    body: TRequestBody,
+  ): Promise<TResponseBody> {
+    return new Promise((resolve, reject) => {
+      const id = this.#messageId++;
+
+      const request: Request = {
+        type: requestType,
+        id,
+        method,
+        body,
+      };
+
+      this.#sendMessage(request, (e: Error) => {
+        if (e) {
+          reject(e);
+          return;
+        }
+
+        let timeout: NodeJS.Timeout | undefined = undefined;
+
+        const messageHandler = (response: Response) => {
+          if (response?.type !== responseType) {
+            return;
+          }
+          if (response.id !== id) {
+            return;
+          }
+
+          if ('error' in response) {
+            const error = new Error(response.error.message);
+            if (response.error.name) {
+              error.name = response.error.name;
+            }
+            reject(error);
+          } else {
+            resolve(response.body as TResponseBody);
+          }
+
+          clearTimeout(timeout);
+          process.removeListener('message', messageHandler);
+        };
+
+        timeout = setTimeout(() => {
+          reject(new Error(`IPC request '${method}' with ID ${id} timed out`));
+          process.removeListener('message', messageHandler);
+        }, 5000);
+        timeout.unref();
+
+        process.addListener('message', messageHandler as () => void);
+      });
+    });
+  }
+}
+
+export const ipcClient = BackstageIpcClient.create();

packages/backend-dev-utils/src/setupTests.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export {};

packages/backend-next/package.json

@@ -18,7 +18,7 @@
     "backstage"
   ],
   "scripts": {
-    "start": "backstage-cli package start",
+    "start": "EXPERIMENTAL_BACKEND_START=1 backstage-cli package start",
     "build": "backstage-cli package build",
     "lint": "backstage-cli package lint",
     "test": "backstage-cli package test",

packages/cli/package.json

@@ -36,6 +36,7 @@
     "@backstage/errors": "workspace:^",
     "@backstage/release-manifests": "workspace:^",
     "@backstage/types": "workspace:^",
+    "@esbuild-kit/cjs-loader": "^2.4.1",
     "@manypkg/get-packages": "^1.1.3",
     "@octokit/request": "^6.0.0",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
@@ -65,6 +66,7 @@
     "chalk": "^4.0.0",
     "chokidar": "^3.3.1",
     "commander": "^9.1.0",
+    "cross-spawn": "^7.0.3",
     "css-loader": "^6.5.1",
     "diff": "^5.0.0",
     "esbuild": "^0.17.0",
@@ -136,6 +138,7 @@
     "@backstage/dev-utils": "workspace:^",
     "@backstage/test-utils": "workspace:^",
     "@backstage/theme": "workspace:^",
+    "@types/cross-spawn": "^6.0.2",
     "@types/diff": "^5.0.0",
     "@types/express": "^4.17.6",
     "@types/fs-extra": "^9.0.1",

packages/cli/src/commands/start/startBackend.ts

@@ -17,6 +17,7 @@
 import fs from 'fs-extra';
 import { paths } from '../../lib/paths';
 import { serveBackend } from '../../lib/bundler';
+import { startBackendExperimental } from '../../lib/experimental/startBackendExperimental';
 
 interface StartBackendOptions {
   checksEnabled: boolean;
@@ -25,17 +26,28 @@ interface StartBackendOptions {
 }
 
 export async function startBackend(options: StartBackendOptions) {
-  // Cleaning dist/ before we start the dev process helps work around an issue
-  // where we end up with the entrypoint executing multiple times, causing
-  // a port bind conflict among other things.
-  await fs.remove(paths.resolveTarget('dist'));
+  if (process.env.EXPERIMENTAL_BACKEND_START) {
+    const waitForExit = await startBackendExperimental({
+      entry: 'src/index',
+      checksEnabled: false, // not supported
+      inspectEnabled: options.inspectEnabled,
+      inspectBrkEnabled: options.inspectBrkEnabled,
+    });
 
-  const waitForExit = await serveBackend({
-    entry: 'src/index',
-    checksEnabled: options.checksEnabled,
-    inspectEnabled: options.inspectEnabled,
-    inspectBrkEnabled: options.inspectBrkEnabled,
-  });
+    await waitForExit();
+  } else {
+    // Cleaning dist/ before we start the dev process helps work around an issue
+    // where we end up with the entrypoint executing multiple times, causing
+    // a port bind conflict among other things.
+    await fs.remove(paths.resolveTarget('dist'));
 
-  await waitForExit();
+    const waitForExit = await serveBackend({
+      entry: 'src/index',
+      checksEnabled: options.checksEnabled,
+      inspectEnabled: options.inspectEnabled,
+      inspectBrkEnabled: options.inspectBrkEnabled,
+    });
+
+    await waitForExit();
+  }
 }

packages/cli/src/lib/experimental/IpcServer.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { serializeError } from '@backstage/errors';
+import { ChildProcess } from 'child_process';
+
+interface RequestMeta {
+  generation: number;
+}
+
+type MethodHandler<TRequest, TResponse> = (
+  req: TRequest,
+  meta: RequestMeta,
+) => Promise<TResponse>;
+
+interface Request {
+  id: number;
+  method: string;
+  body: unknown;
+  type: string;
+}
+
+const requestType = '@backstage/cli/channel/request';
+const responseType = '@backstage/cli/channel/response';
+
+export class IpcServer {
+  #generation = 1;
+  #methods = new Map<string, MethodHandler<any, any>>();
+
+  addChild(child: ChildProcess) {
+    const generation = this.#generation++;
+    const sendMessage = child.send?.bind(child);
+    if (!sendMessage) {
+      return;
+    }
+
+    const messageListener = (request: Request) => {
+      if (request.type !== requestType) {
+        return;
+      }
+
+      const handler = this.#methods.get(request.method);
+      if (!handler) {
+        sendMessage({
+          type: responseType,
+          id: request.id,
+          error: {
+            name: 'NotFoundError',
+            message: `No handler registered for method ${request.method}`,
+          },
+        });
+        return;
+      }
+
+      Promise.resolve()
+        .then(() => handler(request.body, { generation }))
+        .then(response =>
+          sendMessage({
+            type: responseType,
+            id: request.id,
+            body: response,
+          }),
+        )
+        .catch(error =>
+          sendMessage({
+            type: responseType,
+            id: request.id,
+            error: serializeError(error),
+          }),
+        );
+    };
+
+    child.addListener('message', messageListener as (req: unknown) => void);
+
+    child.addListener('exit', () => {
+      child.removeListener('message', messageListener);
+    });
+  }
+
+  registerMethod<TRequest, TResponse>(
+    method: string,
+    handler: MethodHandler<TRequest, TResponse>,
+  ) {
+    if (this.#methods.has(method)) {
+      throw new Error(`A handler is already registered for method ${method}`);
+    }
+    this.#methods.set(method, handler);
+  }
+}

packages/cli/src/lib/experimental/ServerDataStore.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { IpcServer } from './IpcServer';
+
+interface StorageItem {
+  generation: number;
+  data: unknown;
+}
+
+interface SaveRequest {
+  key: string;
+  data: unknown;
+}
+
+interface SaveResponse {
+  saved: boolean;
+}
+
+interface LoadRequest {
+  key: string;
+}
+
+interface LoadResponse {
+  loaded: boolean;
+  data: unknown;
+}
+
+export class ServerDataStore {
+  static bind(server: IpcServer): void {
+    const store = new Map<string, StorageItem>();
+
+    server.registerMethod<SaveRequest, SaveResponse>(
+      'DevDataStore.save',
+      async (request, { generation }) => {
+        const { key, data } = request;
+        if (!key) {
+          throw new Error('Key is required in DevDataStore.save');
+        }
+
+        const item = store.get(key);
+
+        if (!item) {
+          store.set(key, { generation, data });
+          return { saved: true };
+        }
+
+        if (item.generation > generation) {
+          return { saved: false };
+        }
+
+        store.set(key, { generation, data });
+        return { saved: true };
+      },
+    );
+
+    server.registerMethod<LoadRequest, LoadResponse>(
+      'DevDataStore.load',
+      async request => {
+        const item = store.get(request.key);
+        return { loaded: Boolean(item), data: item?.data };
+      },
+    );
+  }
+}

packages/cli/src/lib/experimental/startBackendExperimental.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { BackendServeOptions } from '../bundler/types';
+import type { ChildProcess } from 'child_process';
+import { fileURLToPath } from 'url';
+import { isAbsolute as isAbsolutePath } from 'path';
+import { FSWatcher, watch } from 'chokidar';
+import { IpcServer } from './IpcServer';
+import { ServerDataStore } from './ServerDataStore';
+import debounce from 'lodash/debounce';
+import spawn from 'cross-spawn';
+import { paths } from '../paths';
+
+const loaderArgs = ['--require', require.resolve('@esbuild-kit/cjs-loader')];
+
+export async function startBackendExperimental(options: BackendServeOptions) {
+  const envEnv = process.env as { NODE_ENV: string };
+  if (!envEnv.NODE_ENV) {
+    envEnv.NODE_ENV = 'development';
+  }
+
+  // Set up the parent IPC server and bind the available services
+  const server = new IpcServer();
+  ServerDataStore.bind(server);
+
+  let exiting = false;
+  let child: ChildProcess | undefined;
+  let watcher: FSWatcher | undefined = undefined;
+  let shutdownPromise: Promise<void> | undefined = undefined;
+
+  const restart = debounce(async () => {
+    // If a re-trigger happens during an existing shutdown, we just ignore it
+    if (shutdownPromise) {
+      return;
+    }
+
+    if (child && !child.killed && child.exitCode === null) {
+      // We always wait for the existing process to exit, to make sure we don't get IPC conflicts
+      shutdownPromise = new Promise(resolve => child!.once('exit', resolve));
+      child.kill();
+      await shutdownPromise;
+      shutdownPromise = undefined;
+    }
+
+    // We've received a shutdown signal
+    if (exiting) {
+      return;
+    }
+
+    const optionArgs = new Array<string>();
+    if (options.inspectEnabled) {
+      optionArgs.push('--inspect');
+    } else if (options.inspectBrkEnabled) {
+      optionArgs.push('--inspect-brk');
+    }
+
+    const userArgs = process.argv
+      .slice(['node', 'backstage-cli', 'package', 'start'].length)
+      .filter(arg => !optionArgs.includes(arg));
+
+    child = spawn(
+      process.execPath,
+      [...loaderArgs, ...optionArgs, options.entry, ...userArgs],
+      {
+        stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+        env: {
+          ...process.env,
+          BACKSTAGE_CLI_CHANNEL: '1',
+          ESBK_TSCONFIG_PATH: paths.resolveTargetRoot('tsconfig.json'),
+        },
+        serialization: 'advanced',
+      },
+    );
+
+    server.addChild(child);
+
+    // This captures messages sent by @esbuild-kit/cjs-loader
+    child.on('message', (data: { type?: string } | null) => {
+      if (typeof data === 'object' && data?.type === 'dependency') {
+        let path = (data as { path: string }).path;
+        if (path.startsWith('file:')) {
+          path = fileURLToPath(path);
+        }
+
+        if (isAbsolutePath(path)) {
+          watcher?.add(path);
+        }
+      }
+    });
+  }, 100);
+
+  restart();
+
+  watcher = watch([paths.targetDir], {
+    cwd: process.cwd(),
+    ignored: ['**/.*/**', '**/node_modules/**'],
+    ignoreInitial: true,
+    ignorePermissionErrors: true,
+  }).on('all', restart);
+
+  // Trigger restart on hitting enter in the terminal
+  process.stdin.on('data', restart);
+
+  const exitPromise = new Promise<void>(resolveExitPromise => {
+    async function handleSignal(signal: NodeJS.Signals) {
+      exiting = true;
+
+      // Forward signals to child and wait for it to exit if still running
+      if (child && child.exitCode === null) {
+        await new Promise(resolve => {
+          child!.on('close', resolve);
+          child!.kill(signal);
+        });
+      }
+
+      resolveExitPromise();
+    }
+
+    process.once('SIGINT', handleSignal);
+    process.once('SIGTERM', handleSignal);
+  });
+
+  return () => exitPromise;
+}

packages/cli/src/types.d.ts

@@ -258,3 +258,5 @@ declare module 'webpack-node-externals' {
     }
   }
 }
+
+declare module '@esbuild-kit/cjs-loader' {}

yarn.lock

@@ -3423,6 +3423,7 @@ __metadata:
   resolution: "@backstage/backend-common@workspace:packages/backend-common"
   dependencies:
     "@backstage/backend-app-api": "workspace:^"
+    "@backstage/backend-dev-utils": "workspace:^"
     "@backstage/backend-plugin-api": "workspace:^"
     "@backstage/backend-test-utils": "workspace:^"
     "@backstage/cli": "workspace:^"
@@ -3519,6 +3520,14 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@backstage/backend-dev-utils@workspace:^, @backstage/backend-dev-utils@workspace:packages/backend-dev-utils":
+  version: 0.0.0-use.local
+  resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils"
+  dependencies:
+    "@backstage/cli": "workspace:^"
+  languageName: unknown
+  linkType: soft
+
 "@backstage/backend-plugin-api@workspace:^, @backstage/backend-plugin-api@workspace:packages/backend-plugin-api":
   version: 0.0.0-use.local
   resolution: "@backstage/backend-plugin-api@workspace:packages/backend-plugin-api"
@@ -3664,6 +3673,7 @@ __metadata:
     "@backstage/test-utils": "workspace:^"
     "@backstage/theme": "workspace:^"
     "@backstage/types": "workspace:^"
+    "@esbuild-kit/cjs-loader": ^2.4.1
     "@manypkg/get-packages": ^1.1.3
     "@octokit/request": ^6.0.0
     "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7
@@ -3682,6 +3692,7 @@ __metadata:
     "@swc/core": ^1.3.9
     "@swc/helpers": ^0.4.7
     "@swc/jest": ^0.2.22
+    "@types/cross-spawn": ^6.0.2
     "@types/diff": ^5.0.0
     "@types/express": ^4.17.6
     "@types/fs-extra": ^9.0.1
@@ -3709,6 +3720,7 @@ __metadata:
     chalk: ^4.0.0
     chokidar: ^3.3.1
     commander: ^9.1.0
+    cross-spawn: ^7.0.3
     css-loader: ^6.5.1
     del: ^6.0.0
     diff: ^5.0.0
@@ -9188,6 +9200,26 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@esbuild-kit/cjs-loader@npm:^2.4.1":
+  version: 2.4.1
+  resolution: "@esbuild-kit/cjs-loader@npm:2.4.1"
+  dependencies:
+    "@esbuild-kit/core-utils": ^3.0.0
+    get-tsconfig: ^4.2.0
+  checksum: a516065907be0ead76ac2199ccb08ff92659ba5e2edb4bb8772b6a63afe4faed7eb45a3b4d87266a68c7c135c3dba971cd087bc6f16c382356e835c7dd3440f5
+  languageName: node
+  linkType: hard
+
+"@esbuild-kit/core-utils@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "@esbuild-kit/core-utils@npm:3.0.0"
+  dependencies:
+    esbuild: ~0.15.10
+    source-map-support: ^0.5.21
+  checksum: 0e89ec718e2211bf95c48a8085aaef88e8e416f42abd1c62d488d5458eecd3fbc144179a0c5570ad36fa7e2d3bbc411f8d3fb28802c37ced2154dc2c6ded9dfe
+  languageName: node
+  linkType: hard
+
 "@esbuild/android-arm64@npm:0.17.5":
   version: 0.17.5
   resolution: "@esbuild/android-arm64@npm:0.17.5"
@@ -13942,6 +13974,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/cross-spawn@npm:^6.0.2":
+  version: 6.0.2
+  resolution: "@types/cross-spawn@npm:6.0.2"
+  dependencies:
+    "@types/node": "*"
+  checksum: fa9edd32178878cab3ea8d6d0260639e0fe4860ddb3887b8de53d6e8036e154fc5f313c653f690975aa25025aea8beb83fb0870b931bf8d9202c3ac530a24c9d
+  languageName: node
+  linkType: hard
+
 "@types/d3-color@npm:*":
   version: 3.0.2
   resolution: "@types/d3-color@npm:3.0.2"
@@ -21365,7 +21406,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"esbuild@npm:^0.15.6":
+"esbuild@npm:^0.15.6, esbuild@npm:~0.15.10":
   version: 0.15.18
   resolution: "esbuild@npm:0.15.18"
   dependencies:
@@ -23523,6 +23564,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"get-tsconfig@npm:^4.2.0":
+  version: 4.3.0
+  resolution: "get-tsconfig@npm:4.3.0"
+  checksum: 2597aab99aa3a24db209e192a3e5874ac47fc5abc71703ee26346e0c5816cb346ca09fc813c739db5862d3a2905d89aeca1b0cbc46c2b272398d672309aaf414
+  languageName: node
+  linkType: hard
+
 "getopts@npm:2.3.0":
   version: 2.3.0
   resolution: "getopts@npm:2.3.0"
@@ -35176,7 +35224,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"source-map-support@npm:^0.5.16, source-map-support@npm:~0.5.20":
+"source-map-support@npm:^0.5.16, source-map-support@npm:^0.5.21, source-map-support@npm:~0.5.20":
   version: 0.5.21
   resolution: "source-map-support@npm:0.5.21"
   dependencies: