diff --git a/.changeset/long-chefs-burn.md b/.changeset/long-chefs-burn.md new file mode 100644 index 0000000000..03a7b76db9 --- /dev/null +++ b/.changeset/long-chefs-burn.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Updated database factory to pass service deps required for restoring database state during development. diff --git a/.changeset/old-ways-know.md b/.changeset/old-ways-know.md new file mode 100644 index 0000000000..60a1a923e8 --- /dev/null +++ b/.changeset/old-ways-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +The `DatabaseManager.forPlugin` method now accepts additional service dependencies. There is no need to update existing code to pass these dependencies. diff --git a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts index 8baf84cb69..4d95f863db 100644 --- a/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts +++ b/packages/backend-app-api/src/services/implementations/database/databaseFactory.ts @@ -26,7 +26,8 @@ export const databaseFactory = createServiceFactory({ service: coreServices.database, deps: { config: coreServices.config, - plugin: coreServices.pluginMetadata, + lifecycle: coreServices.lifecycle, + pluginMetadata: coreServices.pluginMetadata, }, async createRootContext({ config }) { return config.getOptional('backend.database') @@ -39,7 +40,10 @@ export const databaseFactory = createServiceFactory({ }), ); }, - async factory({ plugin }, databaseManager) { - return databaseManager.forPlugin(plugin.getId()); + async factory({ pluginMetadata, lifecycle }, databaseManager) { + return databaseManager.forPlugin(pluginMetadata.getId(), { + pluginMetadata, + lifecycle, + }); }, }); diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index b03d6ec866..8b6311f8b0 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -32,6 +32,7 @@ import { IdentityService } from '@backstage/backend-plugin-api'; import { isChildPath } from '@backstage/cli-common'; import { Knex } from 'knex'; import { KubeConfig } from '@kubernetes/client-node'; +import { LifecycleService } from '@backstage/backend-plugin-api'; import { LoadConfigOptionsRemote } from '@backstage/config-loader'; import { Logger } from 'winston'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -40,6 +41,7 @@ import { PermissionsService } from '@backstage/backend-plugin-api'; import { CacheService as PluginCacheManager } from '@backstage/backend-plugin-api'; import { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; import { DiscoveryService as PluginEndpointDiscovery } from '@backstage/backend-plugin-api'; +import { PluginMetadataService } from '@backstage/backend-plugin-api'; import { PushResult } from 'isomorphic-git'; import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; @@ -232,6 +234,7 @@ export class Contexts { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: PluginDatabaseDependencies, ): Knex; // @public @@ -252,7 +255,10 @@ export function createStatusCheckRouter(options: { // @public export class DatabaseManager { - forPlugin(pluginId: string): PluginDatabaseManager; + forPlugin( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): PluginDatabaseManager; static fromConfig( config: Config, options?: DatabaseManagerOptions, @@ -571,6 +577,12 @@ export function notFoundHandler(): RequestHandler; export { PluginCacheManager }; +// @public +export type PluginDatabaseDependencies = { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; +}; + export { PluginDatabaseManager }; export { PluginEndpointDiscovery }; diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c950950680..9f1069f322 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -35,6 +35,7 @@ }, "dependencies": { "@backstage/backend-app-api": "workspace:^", + "@backstage/backend-dev-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/config": "workspace:^", diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 5a497fff29..d8205faec1 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -27,7 +27,7 @@ import { ensureSchemaExists, normalizeConnection, } from './connection'; -import { PluginDatabaseManager } from './types'; +import { PluginDatabaseDependencies, PluginDatabaseManager } from './types'; import path from 'path'; import { LoggerService } from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; @@ -94,12 +94,15 @@ export class DatabaseManager { * should be unique as they are used to look up database config overrides under * `backend.database.plugin`. */ - forPlugin(pluginId: string): PluginDatabaseManager { + forPlugin( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): PluginDatabaseManager { const _this = this; return { getClient(): Promise { - return _this.getDatabase(pluginId); + return _this.getDatabase(pluginId, deps); }, migrations: { skip: false, @@ -307,7 +310,10 @@ export class DatabaseManager { * @returns Promise which resolves to a scoped Knex database client for a * plugin */ - private async getDatabase(pluginId: string): Promise { + private async getDatabase( + pluginId: string, + deps?: PluginDatabaseDependencies, + ): Promise { if (this.databaseCache.has(pluginId)) { return this.databaseCache.get(pluginId)!; } @@ -351,6 +357,7 @@ export class DatabaseManager { const client = createDatabaseClient( pluginConfig, databaseClientOverrides, + deps, ); this.startKeepaliveLoop(pluginId, client); return client; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 5f487886b4..054ab7cfa8 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -19,7 +19,7 @@ import { JsonObject } from '@backstage/types'; import { InputError } from '@backstage/errors'; import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; +import { DatabaseConnector, PluginDatabaseDependencies } from './types'; import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; @@ -55,11 +55,12 @@ const ConnectorMapping: Record = { export function createDatabaseClient( dbConfig: Config, overrides?: Partial, + deps?: PluginDatabaseDependencies, ) { const client: DatabaseClient = dbConfig.getString('client'); return ( - ConnectorMapping[client]?.createClient(dbConfig, overrides) ?? + ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ?? knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) ); } diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 6426005f0e..299e18a111 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -18,8 +18,9 @@ import { Config } from '@backstage/config'; import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; +import { DevDataStore } from '@backstage/backend-dev-utils'; import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; +import { DatabaseConnector, PluginDatabaseDependencies } from '../types'; /** * Creates a knex SQLite3 database connection @@ -30,22 +31,56 @@ import { DatabaseConnector } from '../types'; export function createSqliteDatabaseClient( dbConfig: Config, overrides?: Knex.Config, + deps?: PluginDatabaseDependencies, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); + const connConfig = knexConfig.connection as Knex.Sqlite3ConnectionConfig; // If storage on disk is used, ensure that the directory exists - if ( - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename && - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !== - ':memory:' - ) { - const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; - const directory = path.dirname(filename); - + if (connConfig.filename && connConfig.filename !== ':memory:') { + const directory = path.dirname(connConfig.filename); ensureDirSync(directory); } - const database = knexFactory(knexConfig); + let database: Knex; + + if (deps && connConfig.filename === ':memory:') { + // The dev store is used during watch mode to store and restore the database + // across reloads. It is only available when running the backend through + // `backstage-cli package start`. + const devStore = DevDataStore.get(); + const dataKey = `sqlite3-db-${deps.pluginMetadata.getId()}`; + + if (devStore) { + database = knexFactory({ + ...knexConfig, + connection: async () => { + // If seed data is available, use it to restore the database + const { data: seedData } = await devStore?.load(dataKey); + + return { + ...(knexConfig.connection as Knex.Sqlite3ConnectionConfig), + filename: seedData ?? ':memory:', + }; + }, + }); + } else { + database = knexFactory(knexConfig); + } + + if (devStore) { + // If the dev store is available we save the database state on shutdown + deps?.lifecycle?.addShutdownHook({ + async fn() { + const connection = await database.client.acquireConnection(); + const data = connection.serialize(); + await devStore.save(dataKey, data); + }, + }); + } + } else { + database = knexFactory(knexConfig); + } database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { resource.run('PRAGMA foreign_keys = ON', () => {}); diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 0dde1ec239..483dbd9e40 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -22,5 +22,8 @@ export * from './DatabaseManager'; */ export { createDatabaseClient, ensureDatabaseExists } from './connection'; -export type { PluginDatabaseManager } from './types'; +export type { + PluginDatabaseManager, + PluginDatabaseDependencies, +} from './types'; export { isDatabaseConflictError } from './util'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index 2d59cf636e..7e96c44071 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -14,11 +14,25 @@ * limitations under the License. */ +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { Knex } from 'knex'; export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; +/** + * Service dependencies for `PluginDatabaseManager`. + * + * @public + */ +export type PluginDatabaseDependencies = { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; +}; + /** * DatabaseConnector manages an underlying Knex database driver. */ @@ -26,7 +40,11 @@ export interface DatabaseConnector { /** * createClient provides an instance of a knex database connector. */ - createClient(dbConfig: Config, overrides?: Partial): Knex; + createClient( + dbConfig: Config, + overrides?: Partial, + deps?: PluginDatabaseDependencies, + ): Knex; /** * createNameOverride provides a partial knex config sufficient to override a * database name. diff --git a/yarn.lock b/yarn.lock index 7143443674..650cb4c95d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3423,6 +3423,7 @@ __metadata: resolution: "@backstage/backend-common@workspace:packages/backend-common" dependencies: "@backstage/backend-app-api": "workspace:^" + "@backstage/backend-dev-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" @@ -3519,7 +3520,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/backend-dev-utils@workspace:packages/backend-dev-utils": +"@backstage/backend-dev-utils@workspace:^, @backstage/backend-dev-utils@workspace:packages/backend-dev-utils": version: 0.0.0-use.local resolution: "@backstage/backend-dev-utils@workspace:packages/backend-dev-utils" dependencies: