backend-common: add support for restoring database state via dev store

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

.changeset/long-chefs-burn.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+Updated database factory to pass service deps required for restoring database state during development.

.changeset/old-ways-know.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+The `DatabaseManager.forPlugin` method now accepts additional service dependencies. There is no need to update existing code to pass these dependencies.

packages/backend-app-api/src/services/implementations/database/databaseFactory.ts

@@ -26,7 +26,8 @@ export const databaseFactory = createServiceFactory({
   service: coreServices.database,
   deps: {
     config: coreServices.config,
-    plugin: coreServices.pluginMetadata,
+    lifecycle: coreServices.lifecycle,
+    pluginMetadata: coreServices.pluginMetadata,
   },
   async createRootContext({ config }) {
     return config.getOptional('backend.database')
@@ -39,7 +40,10 @@ export const databaseFactory = createServiceFactory({
           }),
         );
   },
-  async factory({ plugin }, databaseManager) {
-    return databaseManager.forPlugin(plugin.getId());
+  async factory({ pluginMetadata, lifecycle }, databaseManager) {
+    return databaseManager.forPlugin(pluginMetadata.getId(), {
+      pluginMetadata,
+      lifecycle,
+    });
   },
 });

packages/backend-common/api-report.md

@@ -32,6 +32,7 @@ import { IdentityService } from '@backstage/backend-plugin-api';
 import { isChildPath } from '@backstage/cli-common';
 import { Knex } from 'knex';
 import { KubeConfig } from '@kubernetes/client-node';
+import { LifecycleService } from '@backstage/backend-plugin-api';
 import { LoadConfigOptionsRemote } from '@backstage/config-loader';
 import { Logger } from 'winston';
 import { LoggerService } from '@backstage/backend-plugin-api';
@@ -40,6 +41,7 @@ import { PermissionsService } from '@backstage/backend-plugin-api';
 import { CacheService as PluginCacheManager } from '@backstage/backend-plugin-api';
 import { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api';
 import { DiscoveryService as PluginEndpointDiscovery } from '@backstage/backend-plugin-api';
+import { PluginMetadataService } from '@backstage/backend-plugin-api';
 import { PushResult } from 'isomorphic-git';
 import { Readable } from 'stream';
 import { ReadCommitResult } from 'isomorphic-git';
@@ -232,6 +234,7 @@ export class Contexts {
 export function createDatabaseClient(
   dbConfig: Config,
   overrides?: Partial<Knex.Config>,
+  deps?: PluginDatabaseDependencies,
 ): Knex<any, any[]>;
 
 // @public
@@ -252,7 +255,10 @@ export function createStatusCheckRouter(options: {
 
 // @public
 export class DatabaseManager {
-  forPlugin(pluginId: string): PluginDatabaseManager;
+  forPlugin(
+    pluginId: string,
+    deps?: PluginDatabaseDependencies,
+  ): PluginDatabaseManager;
   static fromConfig(
     config: Config,
     options?: DatabaseManagerOptions,
@@ -571,6 +577,12 @@ export function notFoundHandler(): RequestHandler;
 
 export { PluginCacheManager };
 
+// @public
+export type PluginDatabaseDependencies = {
+  lifecycle: LifecycleService;
+  pluginMetadata: PluginMetadataService;
+};
+
 export { PluginDatabaseManager };
 
 export { PluginEndpointDiscovery };

packages/backend-common/package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "@backstage/backend-app-api": "workspace:^",
+    "@backstage/backend-dev-utils": "workspace:^",
     "@backstage/backend-plugin-api": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/config": "workspace:^",

packages/backend-common/src/database/DatabaseManager.ts

@@ -27,7 +27,7 @@ import {
   ensureSchemaExists,
   normalizeConnection,
 } from './connection';
-import { PluginDatabaseManager } from './types';
+import { PluginDatabaseDependencies, PluginDatabaseManager } from './types';
 import path from 'path';
 import { LoggerService } from '@backstage/backend-plugin-api';
 import { stringifyError } from '@backstage/errors';
@@ -94,12 +94,15 @@ export class DatabaseManager {
    * should be unique as they are used to look up database config overrides under
    * `backend.database.plugin`.
    */
-  forPlugin(pluginId: string): PluginDatabaseManager {
+  forPlugin(
+    pluginId: string,
+    deps?: PluginDatabaseDependencies,
+  ): PluginDatabaseManager {
     const _this = this;
 
     return {
       getClient(): Promise<Knex> {
-        return _this.getDatabase(pluginId);
+        return _this.getDatabase(pluginId, deps);
       },
       migrations: {
         skip: false,
@@ -307,7 +310,10 @@ export class DatabaseManager {
    * @returns Promise which resolves to a scoped Knex database client for a
    *          plugin
    */
-  private async getDatabase(pluginId: string): Promise<Knex> {
+  private async getDatabase(
+    pluginId: string,
+    deps?: PluginDatabaseDependencies,
+  ): Promise<Knex> {
     if (this.databaseCache.has(pluginId)) {
       return this.databaseCache.get(pluginId)!;
     }
@@ -351,6 +357,7 @@ export class DatabaseManager {
       const client = createDatabaseClient(
         pluginConfig,
         databaseClientOverrides,
+        deps,
       );
       this.startKeepaliveLoop(pluginId, client);
       return client;

packages/backend-common/src/database/connection.ts

@@ -19,7 +19,7 @@ import { JsonObject } from '@backstage/types';
 import { InputError } from '@backstage/errors';
 import knexFactory, { Knex } from 'knex';
 import { mergeDatabaseConfig } from './config';
-import { DatabaseConnector } from './types';
+import { DatabaseConnector, PluginDatabaseDependencies } from './types';
 
 import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors';
 
@@ -55,11 +55,12 @@ const ConnectorMapping: Record<DatabaseClient, DatabaseConnector> = {
 export function createDatabaseClient(
   dbConfig: Config,
   overrides?: Partial<Knex.Config>,
+  deps?: PluginDatabaseDependencies,
 ) {
   const client: DatabaseClient = dbConfig.getString('client');
 
   return (
-    ConnectorMapping[client]?.createClient(dbConfig, overrides) ??
+    ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ??
     knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides))
   );
 }

packages/backend-common/src/database/connectors/sqlite3.ts

@@ -18,8 +18,9 @@ import { Config } from '@backstage/config';
 import { ensureDirSync } from 'fs-extra';
 import knexFactory, { Knex } from 'knex';
 import path from 'path';
+import { DevDataStore } from '@backstage/backend-dev-utils';
 import { mergeDatabaseConfig } from '../config';
-import { DatabaseConnector } from '../types';
+import { DatabaseConnector, PluginDatabaseDependencies } from '../types';
 
 /**
  * Creates a knex SQLite3 database connection
@@ -30,22 +31,56 @@ import { DatabaseConnector } from '../types';
 export function createSqliteDatabaseClient(
   dbConfig: Config,
   overrides?: Knex.Config,
+  deps?: PluginDatabaseDependencies,
 ) {
   const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides);
+  const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
 
   // If storage on disk is used, ensure that the directory exists
-  if (
-    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename &&
-    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !==
-      ':memory:'
-  ) {
-    const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
-    const directory = path.dirname(filename);
-
+  if (connConfig.filename && connConfig.filename !== ':memory:') {
+    const directory = path.dirname(connConfig.filename);
     ensureDirSync(directory);
   }
 
-  const database = knexFactory(knexConfig);
+  let database: Knex;
+
+  if (deps && connConfig.filename === ':memory:') {
+    // The dev store is used during watch mode to store and restore the database
+    // across reloads. It is only available when running the backend through
+    // `backstage-cli package start`.
+    const devStore = DevDataStore.get();
+    const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`;
+
+    if (devStore) {
+      database = knexFactory({
+        ...knexConfig,
+        connection: async () => {
+          // If seed data is available, use it to restore the database
+          const { data: seedData } = await devStore?.load(dataKey);
+
+          return {
+            ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig),
+            filename: seedData ?? ':memory:',
+          };
+        },
+      });
+    } else {
+      database = knexFactory(knexConfig);
+    }
+
+    if (devStore) {
+      // If the dev store is available we save the database state on shutdown
+      deps?.lifecycle?.addShutdownHook({
+        async fn() {
+          const connection = await database.client.acquireConnection();
+          const data = connection.serialize();
+          await devStore.save(dataKey, data);
+        },
+      });
+    }
+  } else {
+    database = knexFactory(knexConfig);
+  }
 
   database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
     resource.run('PRAGMA foreign_keys = ON', () => {});

packages/backend-common/src/database/index.ts

@@ -22,5 +22,8 @@ export * from './DatabaseManager';
  */
 export { createDatabaseClient, ensureDatabaseExists } from './connection';
 
-export type { PluginDatabaseManager } from './types';
+export type {
+  PluginDatabaseManager,
+  PluginDatabaseDependencies,
+} from './types';
 export { isDatabaseConflictError } from './util';

packages/backend-common/src/database/types.ts

@@ -14,11 +14,25 @@
  * limitations under the License.
  */
 
+import {
+  LifecycleService,
+  PluginMetadataService,
+} from '@backstage/backend-plugin-api';
 import { Config } from '@backstage/config';
 import { Knex } from 'knex';
 
 export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api';
 
+/**
+ * Service dependencies for `PluginDatabaseManager`.
+ *
+ * @public
+ */
+export type PluginDatabaseDependencies = {
+  lifecycle: LifecycleService;
+  pluginMetadata: PluginMetadataService;
+};
+
 /**
  * DatabaseConnector manages an underlying Knex database driver.
  */
@@ -26,7 +40,11 @@ export interface DatabaseConnector {
   /**
    * createClient provides an instance of a knex database connector.
    */
-  createClient(dbConfig: Config, overrides?: Partial<Knex.Config>): Knex;
+  createClient(
+    dbConfig: Config,
+    overrides?: Partial<Knex.Config>,
+    deps?: PluginDatabaseDependencies,
+  ): Knex;
   /**
    * createNameOverride provides a partial knex config sufficient to override a
    * database name.

yarn.lock

@@ -3423,6 +3423,7 @@ __metadata:
   resolution: "@backstage/backend-common@workspace:packages/backend-common"
   dependencies:
     "@backstage/backend-app-api": "workspace:^"
+    "@backstage/backend-dev-utils": "workspace:^"
     "@backstage/backend-plugin-api": "workspace:^"
     "@backstage/backend-test-utils": "workspace:^"
     "@backstage/cli": "workspace:^"
@@ -3519,7 +3520,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@backstage/backend-dev-utils@workspace:packages/backend-dev-utils":
+"@backstage/backend-dev-utils@workspace:^, @backstage/backend-dev-utils@workspace:packages/backend-dev-utils":
   version: 0.0.0-use.local
   resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils"
   dependencies: