diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts index bf4af17d05..6a461fad55 100644 --- a/packages/backend-common/src/database/DatabaseManager.test.ts +++ b/packages/backend-common/src/database/DatabaseManager.test.ts @@ -16,12 +16,12 @@ import { ConfigReader } from '@backstage/config'; import { omit } from 'lodash'; import path from 'path'; +import { DatabaseManager } from './DatabaseManager'; import { createDatabaseClient, ensureDatabaseExists, ensureSchemaExists, } from './connection'; -import { DatabaseManager } from './DatabaseManager'; jest.mock('./connection', () => ({ ...jest.requireActual('./connection'), diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 776f05aed4..a261cd13fc 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -14,23 +14,19 @@ * limitations under the License. */ -import { Config, ConfigReader } from '@backstage/config'; -import { JsonObject } from '@backstage/types'; -import { Knex } from 'knex'; -import { merge, omit } from 'lodash'; -import { mergeDatabaseConfig } from './config'; -import { PluginDatabaseManager } from './types'; -import path from 'path'; import { DatabaseService, LifecycleService, LoggerService, PluginMetadataService, } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; import { stringifyError } from '@backstage/errors'; +import { Knex } from 'knex'; +import { MysqlConnector } from './connectors/mysql'; import { PgConnector } from './connectors/postgres'; import { Sqlite3Connector } from './connectors/sqlite3'; -import { MysqlConnector } from './connectors/mysql'; +import { Connector, PluginDatabaseManager } from './types'; /** * Provides a config lookup path for a plugin's config block. @@ -81,15 +77,15 @@ export class DatabaseManager implements LegacyRootDatabaseService { options?: DatabaseManagerOptions, ): DatabaseManager { const databaseConfig = config.getConfig('backend.database'); + const prefix = databaseConfig.getOptionalString('prefix'); return new DatabaseManager( databaseConfig, - databaseConfig.getOptionalString('prefix'), { - pg: new PgConnector(config, prefix, options), - sqlite3: new Sqlite3Connector(config, prefix, options), - 'better-sqlite3': new Sqlite3Connector(config, prefix, options), - mysql: new MysqlConnector(config, prefix, options), - mysql2: new MysqlConnector(config, prefix, options), + pg: new PgConnector(config, prefix), + sqlite3: new Sqlite3Connector(config, prefix), + 'better-sqlite3': new Sqlite3Connector(config, prefix), + mysql: new MysqlConnector(config, prefix), + mysql2: new MysqlConnector(config, prefix), }, options, ); @@ -97,19 +93,7 @@ export class DatabaseManager implements LegacyRootDatabaseService { private constructor( private readonly config: Config, - private readonly prefix: string = 'backstage_plugin_', - private readonly connectors: Record< - string, - { - getClient( - pluginId: string, - deps?: { - lifecycle: LifecycleService; - pluginMetadata: PluginMetadataService; - }, - ): Promise; - } - >, + private readonly connectors: Record, private readonly options?: DatabaseManagerOptions, private readonly databaseCache: Map> = new Map(), ) {} @@ -131,54 +115,15 @@ export class DatabaseManager implements LegacyRootDatabaseService { const client = this.getClientType(pluginId).client; const connector = this.connectors[client]; if (!connector) { - throw new Error(`Unsupported database client type '${client}'`); + throw new Error( + `Unsupported database client type '${client}' specified for plugin '${pluginId}'`, + ); } - const getClient = () => this.getDatabase(pluginId, deps); + const getClient = () => this.getDatabase(pluginId, connector, deps); const migrations = { skip: false, ...this.options?.migrations }; return { getClient, migrations }; } - /** - * Provides the canonical database name for a given plugin. - * - * This method provides the effective database name which is determined using global - * and plugin specific database config. If no explicit database name is configured - * and `pluginDivisionMode` is not `schema`, this method will provide a generated name - * which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is - * `schema`, it will fallback to using the default database for the knex instance. - * - * @param pluginId - Lookup the database name for given plugin - * @returns String representing the plugin's database name - */ - private getDatabaseName(pluginId: string): string | undefined { - const connection = this.getConnectionConfig(pluginId); - - if (this.getClientType(pluginId).client.includes('sqlite3')) { - const sqliteFilename: string | undefined = ( - connection as Knex.Sqlite3ConnectionConfig - ).filename; - - if (sqliteFilename === ':memory:') { - return sqliteFilename; - } - - const sqliteDirectory = - (connection as { directory?: string }).directory ?? '.'; - - return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`); - } - - const databaseName = (connection as Knex.ConnectionConfig)?.database; - - // `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database - if (this.getPluginDivisionModeConfig() === 'schema') { - return databaseName; - } - - // all other supported databases should fallback to an auto-prefixed name - return databaseName ?? `${this.prefix}${pluginId}`; - } - /** * Provides the client type which should be used for a given plugin. * @@ -206,143 +151,6 @@ export class DatabaseManager implements LegacyRootDatabaseService { }; } - private getRoleConfig(pluginId: string): string | undefined { - return ( - this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ?? - this.config.getOptionalString('role') - ); - } - - /** - * Provides the knexConfig which should be used for a given plugin. - * - * @param pluginId - Plugin to get the knexConfig for - * @returns The merged knexConfig value or undefined if it isn't specified - */ - private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined { - const pluginConfig = this.config - .getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`) - ?.get(); - - const baseConfig = this.config - .getOptionalConfig('knexConfig') - ?.get(); - - return merge(baseConfig, pluginConfig); - } - - private getEnsureExistsConfig(pluginId: string): boolean { - const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true; - return ( - this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ?? - baseConfig - ); - } - - private getPluginDivisionModeConfig(): string { - return this.config.getOptionalString('pluginDivisionMode') ?? 'database'; - } - - /** - * Provides a Knex connection plugin config by combining base and plugin - * config. - * - * This method provides a baseConfig for a plugin database connector. If the - * client type has not been overridden, the global connection config will be - * included with plugin specific config as the base. Values from the plugin - * connection take precedence over the base. Base database name is omitted for - * all supported databases excluding SQLite unless `pluginDivisionMode` is set - * to `schema`. - */ - private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig { - const { client, overridden } = this.getClientType(pluginId); - - let baseConnection = normalizeConnection( - this.config.get('connection'), - this.config.getString('client'), - ); - - if ( - client.includes('sqlite3') && - 'filename' in baseConnection && - baseConnection.filename !== ':memory:' - ) { - throw new Error( - '`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.', - ); - } - - // Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The - // `database` property from the base connection is omitted unless `pluginDivisionMode` - // is set to `schema`. SQLite3's `filename` property is an exception as this is used as a - // directory elsewhere so we preserve `filename`. - if (this.getPluginDivisionModeConfig() !== 'schema') { - baseConnection = omit(baseConnection, 'database'); - } - - // get and normalize optional plugin specific database connection - const connection = normalizeConnection( - this.config.getOptional(`${pluginPath(pluginId)}.connection`), - client, - ); - - if (client === 'pg') { - ( - baseConnection as Knex.PgConnectionConfig - ).application_name ||= `backstage_plugin_${pluginId}`; - } - - return { - // include base connection if client type has not been overridden - ...(overridden ? {} : baseConnection), - ...connection, - } as Knex.StaticConnectionConfig; - } - - /** - * Provides a Knex database config for a given plugin. - * - * This method provides a Knex configuration object along with the plugin's - * client type. - * - * @param pluginId - The plugin that the database config should correspond with - */ - private getConfigForPlugin(pluginId: string): Knex.Config { - const { client } = this.getClientType(pluginId); - const role = this.getRoleConfig(pluginId); - - return { - ...this.getAdditionalKnexConfig(pluginId), - client, - connection: this.getConnectionConfig(pluginId), - ...(role && { role }), - }; - } - - /** - * Provides a partial `Knex.Config` database schema override for a given - * plugin. - * - * @param pluginId - Target plugin to get database schema override - * @returns Partial `Knex.Config` with database schema override - */ - private getSchemaOverrides(pluginId: string): Knex.Config | undefined { - return createSchemaOverride(this.getClientType(pluginId).client, pluginId); - } - - /** - * Provides a partial `Knex.Config`• database name override for a given plugin. - * - * @param pluginId - Target plugin to get database name override - * @returns Partial `Knex.Config` with database name override - */ - private getDatabaseOverrides(pluginId: string): Knex.Config { - const databaseName = this.getDatabaseName(pluginId); - return databaseName - ? createNameOverride(this.getClientType(pluginId).client, databaseName) - : {}; - } - /** * Provides a scoped Knex client for a plugin as per application config. * @@ -352,6 +160,7 @@ export class DatabaseManager implements LegacyRootDatabaseService { */ private async getDatabase( pluginId: string, + connector: Connector, deps?: { lifecycle: LifecycleService; pluginMetadata: PluginMetadataService; @@ -361,55 +170,13 @@ export class DatabaseManager implements LegacyRootDatabaseService { return this.databaseCache.get(pluginId)!; } - const clientPromise = Promise.resolve().then(async () => { - const pluginConfig = new ConfigReader( - this.getConfigForPlugin(pluginId) as JsonObject, - ); - - const databaseName = this.getDatabaseName(pluginId); - if (databaseName && this.getEnsureExistsConfig(pluginId)) { - try { - await ensureDatabaseExists(pluginConfig, databaseName); - } catch (error) { - throw new Error( - `Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`, - ); - } - } - - let schemaOverrides; - if (this.getPluginDivisionModeConfig() === 'schema') { - schemaOverrides = this.getSchemaOverrides(pluginId); - if (this.getEnsureExistsConfig(pluginId)) { - try { - await ensureSchemaExists(pluginConfig, pluginId); - } catch (error) { - throw new Error( - `Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`, - ); - } - } - } - - const databaseClientOverrides = mergeDatabaseConfig( - {}, - this.getDatabaseOverrides(pluginId), - schemaOverrides, - ); - - const client = createDatabaseClient( - pluginConfig, - databaseClientOverrides, - deps, - ); - if (process.env.NODE_ENV !== 'test') { - this.startKeepaliveLoop(pluginId, client); - } - return client; - }); - + const clientPromise = connector.getClient(pluginId, deps).then(); this.databaseCache.set(pluginId, clientPromise); + if (process.env.NODE_ENV !== 'test') { + clientPromise.then(client => this.startKeepaliveLoop(pluginId, client)); + } + return clientPromise; } diff --git a/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts b/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts index 1da8e6c11e..59f7b5b137 100644 --- a/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts +++ b/packages/backend-common/src/database/connectors/defaultNameOverride.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import defaultNameOverride from './defaultNameOverride'; describe('defaultNameOverride()', () => { diff --git a/packages/backend-common/src/database/connectors/defaultNameOverride.ts b/packages/backend-common/src/database/connectors/defaultNameOverride.ts index bc467eb7f6..3f610c51c6 100644 --- a/packages/backend-common/src/database/connectors/defaultNameOverride.ts +++ b/packages/backend-common/src/database/connectors/defaultNameOverride.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Knex } from 'knex'; /** diff --git a/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts b/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts index 2db5ecd454..505b9051f7 100644 --- a/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts +++ b/packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import defaultSchemaOverride from './defaultSchemaOverride'; describe('defaultNameOverride()', () => { diff --git a/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts b/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts index 4e76308542..85c6f1ec65 100644 --- a/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts +++ b/packages/backend-common/src/database/connectors/defaultSchemaOverride.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Knex } from 'knex'; /** diff --git a/packages/backend-common/src/database/connectors/index.ts b/packages/backend-common/src/database/connectors/index.ts index df84ec66ba..9d18e7460a 100644 --- a/packages/backend-common/src/database/connectors/index.ts +++ b/packages/backend-common/src/database/connectors/index.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export * from './mysql'; export * from './postgres'; export * from './sqlite3'; diff --git a/packages/backend-common/src/database/config.test.ts b/packages/backend-common/src/database/connectors/mergeDatabaseConfig.test.ts similarity index 97% rename from packages/backend-common/src/database/config.test.ts rename to packages/backend-common/src/database/connectors/mergeDatabaseConfig.test.ts index a5bf4a79d2..84177a1208 100644 --- a/packages/backend-common/src/database/config.test.ts +++ b/packages/backend-common/src/database/connectors/mergeDatabaseConfig.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import { mergeDatabaseConfig } from './config'; +import { mergeDatabaseConfig } from './mergeDatabaseConfig'; -describe('config', () => { +describe('mergeDatabaseConfig', () => { describe('mergeDatabaseConfig', () => { it('does not mutate the input object', () => { const input = { diff --git a/packages/backend-common/src/database/config.ts b/packages/backend-common/src/database/connectors/mergeDatabaseConfig.ts similarity index 100% rename from packages/backend-common/src/database/config.ts rename to packages/backend-common/src/database/connectors/mergeDatabaseConfig.ts diff --git a/packages/backend-common/src/database/connectors/mysql.ts b/packages/backend-common/src/database/connectors/mysql.ts index 63464fa6f4..a7cbad271f 100644 --- a/packages/backend-common/src/database/connectors/mysql.ts +++ b/packages/backend-common/src/database/connectors/mysql.ts @@ -25,9 +25,9 @@ import knexFactory, { Knex } from 'knex'; import { merge, omit } from 'lodash'; import path from 'path'; import yn from 'yn'; -import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; +import { Connector, DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; +import { mergeDatabaseConfig } from './mergeDatabaseConfig'; /** * Creates a knex mysql database connection @@ -295,7 +295,7 @@ function createNameOverride( } } -export class MysqlConnector { +export class MysqlConnector implements Connector { constructor( private readonly config: Config, private readonly prefix: string = 'backstage_plugin_', diff --git a/packages/backend-common/src/database/connectors/postgres.test.ts b/packages/backend-common/src/database/connectors/postgres.test.ts index 39c9af5ef3..1e07984967 100644 --- a/packages/backend-common/src/database/connectors/postgres.test.ts +++ b/packages/backend-common/src/database/connectors/postgres.test.ts @@ -16,8 +16,8 @@ import { Config, ConfigReader } from '@backstage/config'; import { - createPgDatabaseClient, buildPgDatabaseConfig, + createPgDatabaseClient, getPgConnectionConfig, parsePgConnectionString, } from './postgres'; diff --git a/packages/backend-common/src/database/connectors/postgres.ts b/packages/backend-common/src/database/connectors/postgres.ts index 0a3aa6cb74..4e3ed9aa1d 100644 --- a/packages/backend-common/src/database/connectors/postgres.ts +++ b/packages/backend-common/src/database/connectors/postgres.ts @@ -25,10 +25,10 @@ import knexFactory, { Knex } from 'knex'; import { merge, omit } from 'lodash'; import path from 'path'; import { Client } from 'pg'; -import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; +import { Connector, DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; import defaultSchemaOverride from './defaultSchemaOverride'; +import { mergeDatabaseConfig } from './mergeDatabaseConfig'; /** * Creates a knex postgres database connection @@ -292,7 +292,7 @@ function createNameOverride( } } -export class PgConnector { +export class PgConnector implements Connector { constructor( private readonly config: Config, private readonly prefix: string = 'backstage_plugin_', diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 5714b036e3..514af6d69f 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -26,8 +26,8 @@ import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import { merge, omit } from 'lodash'; import path from 'path'; -import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; +import { Connector, DatabaseConnector } from '../types'; +import { mergeDatabaseConfig } from './mergeDatabaseConfig'; /** * Creates a knex SQLite3 database connection @@ -227,7 +227,7 @@ function createNameOverride( } } -export class Sqlite3Connector implements DatabaseConnector { +export class Sqlite3Connector implements Connector { constructor( private readonly config: Config, private readonly prefix: string = 'backstage_plugin_', diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 330429f0ef..f8bdc22f6d 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -22,8 +22,8 @@ export * from './DatabaseManager'; */ export { createDatabaseClient, - ensureDatabaseExists, dropDatabase, + ensureDatabaseExists, } from './connection'; export type { PluginDatabaseManager } from './types'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index c73540a9c1..967de6966e 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -86,3 +86,13 @@ export interface DatabaseConnector { */ dropDatabase?(dbConfig: Config, ...databases: Array): Promise; } + +export interface Connector { + getClient( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise; +}