From 1da71d800c3e0c7e86c53755844d71b116eacf3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 28 Apr 2024 11:33:59 +0200 Subject: [PATCH] Make individual connector classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/database/DatabaseManager.ts | 36 +- .../src/database/connection.test.ts | 308 --------------- .../backend-common/src/database/connection.ts | 355 +++++++++--------- .../src/database/connectors/mysql.ts | 336 ++++++++++++++++- .../src/database/connectors/postgres.ts | 341 ++++++++++++++++- .../src/database/connectors/sqlite3.ts | 339 ++++++++++++++++- packages/backend-common/src/database/types.ts | 28 +- 7 files changed, 1224 insertions(+), 519 deletions(-) delete mode 100644 packages/backend-common/src/database/connection.test.ts diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 8a4a4b50fa..776f05aed4 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -19,14 +19,6 @@ import { JsonObject } from '@backstage/types'; import { Knex } from 'knex'; import { merge, omit } from 'lodash'; import { mergeDatabaseConfig } from './config'; -import { - createDatabaseClient, - createNameOverride, - createSchemaOverride, - ensureDatabaseExists, - ensureSchemaExists, - normalizeConnection, -} from './connection'; import { PluginDatabaseManager } from './types'; import path from 'path'; import { @@ -36,6 +28,9 @@ import { PluginMetadataService, } from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; +import { PgConnector } from './connectors/postgres'; +import { Sqlite3Connector } from './connectors/sqlite3'; +import { MysqlConnector } from './connectors/mysql'; /** * Provides a config lookup path for a plugin's config block. @@ -86,10 +81,16 @@ export class DatabaseManager implements LegacyRootDatabaseService { options?: DatabaseManagerOptions, ): DatabaseManager { const databaseConfig = config.getConfig('backend.database'); - return new DatabaseManager( databaseConfig, databaseConfig.getOptionalString('prefix'), + { + pg: new PgConnector(config, prefix, options), + sqlite3: new Sqlite3Connector(config, prefix, options), + 'better-sqlite3': new Sqlite3Connector(config, prefix, options), + mysql: new MysqlConnector(config, prefix, options), + mysql2: new MysqlConnector(config, prefix, options), + }, options, ); } @@ -97,6 +98,18 @@ export class DatabaseManager implements LegacyRootDatabaseService { private constructor( private readonly config: Config, private readonly prefix: string = 'backstage_plugin_', + private readonly connectors: Record< + string, + { + getClient( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise; + } + >, private readonly options?: DatabaseManagerOptions, private readonly databaseCache: Map> = new Map(), ) {} @@ -115,6 +128,11 @@ export class DatabaseManager implements LegacyRootDatabaseService { pluginMetadata: PluginMetadataService; }, ): PluginDatabaseManager { + const client = this.getClientType(pluginId).client; + const connector = this.connectors[client]; + if (!connector) { + throw new Error(`Unsupported database client type '${client}'`); + } const getClient = () => this.getDatabase(pluginId, deps); const migrations = { skip: false, ...this.options?.migrations }; return { getClient, migrations }; diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts deleted file mode 100644 index 3028d46cae..0000000000 --- a/packages/backend-common/src/database/connection.test.ts +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ConfigReader } from '@backstage/config'; -import { - createDatabaseClient, - createNameOverride, - createSchemaOverride, - dropDatabase, - ensureSchemaExists, - parseConnectionString, -} from './connection'; -import { mysqlConnector, pgConnector } from './connectors'; - -const mocked = (f: Function) => f as jest.Mock; - -jest.mock('./connectors', () => { - const connectors = jest.requireActual('./connectors'); - return { - ...connectors, - mysqlConnector: { - ...connectors.mysqlConnector, - dropDatabase: jest.fn(), - }, - pgConnector: { - ...connectors.pgConnector, - dropDatabase: jest.fn(), - ensureSchemaExists: jest.fn(), - }, - }; -}); - -describe('database connection', () => { - describe('createDatabaseClient', () => { - it('returns a postgres connection', () => { - expect( - createDatabaseClient( - new ConfigReader({ - client: 'pg', - connection: { - host: 'acme', - user: 'foo', - password: 'bar', - database: 'foodb', - }, - }), - ), - ).toBeTruthy(); - }); - - it('returns an sqlite connection', () => { - expect( - createDatabaseClient( - new ConfigReader({ - client: 'better-sqlite3', - connection: ':memory:', - }), - ), - ).toBeTruthy(); - }); - - it('returns a mysql connection', () => { - expect(() => - createDatabaseClient( - new ConfigReader({ - client: 'mysql2', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - ), - ).toBeTruthy(); - }); - - it('accepts overrides', () => { - expect( - createDatabaseClient( - new ConfigReader({ - client: 'pg', - connection: { - host: 'acme', - user: 'foo', - password: 'bar', - database: 'foodb', - }, - }), - { - connection: { - database: 'foo', - }, - }, - ), - ).toBeTruthy(); - }); - - it('throws an error without a client', () => { - expect(() => - createDatabaseClient( - new ConfigReader({ - connection: '', - }), - ), - ).toThrow(); - }); - - it('throws an error without a connection', () => { - expect(() => - createDatabaseClient( - new ConfigReader({ - client: 'pg', - }), - ), - ).toThrow(); - }); - }); - - describe('createNameOverride', () => { - it('returns Knex config for postgres', () => { - expect(createNameOverride('pg', 'testpg')).toHaveProperty( - 'connection.database', - 'testpg', - ); - }); - - it('returns Knex config for sqlite', () => { - expect(createNameOverride('better-sqlite3', 'testsqlite')).toHaveProperty( - 'connection.filename', - 'testsqlite', - ); - }); - - it('returns Knex config for mysql', () => { - expect(createNameOverride('mysql', 'testmysql')).toHaveProperty( - 'connection.database', - 'testmysql', - ); - }); - - it('throws an error for unknown connection', () => { - expect(() => createNameOverride('unknown', 'testname')).toThrow(); - }); - }); - - describe('parseConnectionString', () => { - it('returns parsed Knex.StaticConnectionConfig for postgres', () => { - expect( - parseConnectionString('postgresql://foo:bar@acme:5432/foodb', 'pg'), - ).toHaveProperty('database', 'foodb'); - }); - - it('returns parsed Knex.StaticConnectionConfig for mysql2', () => { - expect( - parseConnectionString('mysql://foo:bar@acme:3306/foodb', 'mysql2'), - ).toHaveProperty('database', 'foodb'); - }); - - it('throws an error if client hint is not provided', () => { - expect(() => parseConnectionString('sqlite://')).toThrow(); - }); - }); - - describe('createSchemaOverride', () => { - it('returns Knex config for postgres', () => { - expect(createSchemaOverride('pg', 'testpg')).toHaveProperty( - 'searchPath', - ['testpg'], - ); - }); - - it('throws error for sqlite', () => { - expect( - createSchemaOverride('better-sqlite3', 'testsqlite'), - ).toBeUndefined(); - }); - - it('returns Knex config for mysql', () => { - expect(createSchemaOverride('mysql', 'testmysql')).toBeUndefined(); - }); - - it('throws an error for unknown connection', () => { - expect(createSchemaOverride('unknown', 'testname')).toBeUndefined(); - }); - }); - - describe('ensureSchemaExists', () => { - it('returns successfully with pg client', async () => { - await ensureSchemaExists( - new ConfigReader({ - client: 'pg', - schema: 'catalog', - connection: 'postgresql://testuser:testpass@acme:5432/userdbname', - }), - 'catalog', - ); - - const mockCalls = mocked( - pgConnector.ensureSchemaExists as Function, - ).mock.calls.splice(-1); - const [baseConfig, schemaName] = mockCalls[0]; - - expect(baseConfig.get()).toMatchObject({ - client: 'pg', - connection: 'postgresql://testuser:testpass@acme:5432/userdbname', - }); - - expect(schemaName).toEqual('catalog'); - }); - - it('throws error for non pg client', () => { - return expect( - ensureSchemaExists( - new ConfigReader({ - client: 'better-sqlite3', - schema: 'catalog', - connection: ':memory:', - }), - 'catalog', - ), - ).resolves.toBeUndefined(); - }); - }); - - describe('dropDatabase', () => { - it('returns successfully with pg client', async () => { - await dropDatabase( - new ConfigReader({ - client: 'pg', - schema: 'catalog', - connection: 'postgresql://testuser:testpass@acme:5432/userdbname', - }), - 'backstage_plugin_foobar', - ); - - const mockCalls = mocked( - pgConnector.dropDatabase as Function, - ).mock.calls.splice(-1); - const [baseConfig, databaseName] = mockCalls[0]; - - expect(baseConfig.get()).toMatchObject({ - client: 'pg', - connection: 'postgresql://testuser:testpass@acme:5432/userdbname', - }); - - expect(databaseName).toEqual('backstage_plugin_foobar'); - }); - - it('returns successfully with mysql client', async () => { - await dropDatabase( - new ConfigReader({ - client: 'mysql2', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - 'backstage_plugin_foobar', - ); - - const mockCalls = mocked( - mysqlConnector.dropDatabase as Function, - ).mock.calls.splice(-1); - const [baseConfig, databaseName] = mockCalls[0]; - - expect(baseConfig.get()).toMatchObject({ - client: 'mysql2', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }); - - expect(databaseName).toEqual('backstage_plugin_foobar'); - }); - - it('does nothing in other database drivers', () => { - return expect( - dropDatabase( - new ConfigReader({ - client: 'better-sqlite3', - schema: 'catalog', - connection: ':memory:', - }), - 'catalog', - ), - ).resolves.toBeUndefined(); - }); - }); -}); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index c0e031516f..875734aa19 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,190 +14,205 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import { JsonObject } from '@backstage/types'; -import { InputError } from '@backstage/errors'; -import knexFactory, { Knex } from 'knex'; -import limiterFactory from 'p-limit'; -import { mergeDatabaseConfig } from './config'; -import { DatabaseConnector } from './types'; +// * Copyright 2020 The Backstage Authors +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ -import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; -import { - LifecycleService, - PluginMetadataService, -} from '@backstage/backend-plugin-api'; +// import { Config } from '@backstage/config'; +// import { JsonObject } from '@backstage/types'; +// import { InputError } from '@backstage/errors'; +// import knexFactory, { Knex } from 'knex'; +// import limiterFactory from 'p-limit'; +// import { mergeDatabaseConfig } from './config'; +// import { DatabaseConnector } from './types'; -type DatabaseClient = - | 'pg' - | 'better-sqlite3' - | 'sqlite3' - | 'mysql' - | 'mysql2' - | string; +// import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; +// import { +// LifecycleService, +// PluginMetadataService, +// } from '@backstage/backend-plugin-api'; -// This limits the number of concurrent CREATE DATABASE and CREATE SCHEMA -// commands, globally, to just one. This is overly defensive, and was added as -// an attempt to counteract the pool issues on recent node versions. See -// https://github.com/backstage/backstage/pull/19988 -const ddlLimiter = limiterFactory(1); +// type DatabaseClient = +// | 'pg' +// | 'better-sqlite3' +// | 'sqlite3' +// | 'mysql' +// | 'mysql2' +// | string; -/** - * Mapping of client type to supported database connectors - * - * Database connectors can be aliased here, for example mysql2 uses - * the same connector as mysql. - */ -const ConnectorMapping: Record = { - pg: pgConnector, - 'better-sqlite3': sqlite3Connector, - sqlite3: sqlite3Connector, - mysql: mysqlConnector, - mysql2: mysqlConnector, -}; +// // This limits the number of concurrent CREATE DATABASE and CREATE SCHEMA +// // commands, globally, to just one. This is overly defensive, and was added as +// // an attempt to counteract the pool issues on recent node versions. See +// // https://github.com/backstage/backstage/pull/19988 +// const ddlLimiter = limiterFactory(1); -/** - * Creates a knex database connection - * - * @public - * @param dbConfig - The database config - * @param overrides - Additional options to merge with the config - */ -export function createDatabaseClient( - dbConfig: Config, - overrides?: Partial, - deps?: { - lifecycle: LifecycleService; - pluginMetadata: PluginMetadataService; - }, -) { - const client: DatabaseClient = dbConfig.getString('client'); +// /** +// * Mapping of client type to supported database connectors +// * +// * Database connectors can be aliased here, for example mysql2 uses +// * the same connector as mysql. +// */ +// const ConnectorMapping: Record = { +// pg: pgConnector, +// 'better-sqlite3': sqlite3Connector, +// sqlite3: sqlite3Connector, +// mysql: mysqlConnector, +// mysql2: mysqlConnector, +// }; - return ( - ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ?? - knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) - ); -} +// /** +// * Creates a knex database connection +// * +// * @public +// * @param dbConfig - The database config +// * @param overrides - Additional options to merge with the config +// */ +// export function createDatabaseClient( +// dbConfig: Config, +// overrides?: Partial, +// deps?: { +// lifecycle: LifecycleService; +// pluginMetadata: PluginMetadataService; +// }, +// ) { +// const client: DatabaseClient = dbConfig.getString('client'); -/** - * Ensures that the given databases all exist, creating them if they do not. - * - * @public - */ -export async function ensureDatabaseExists( - dbConfig: Config, - ...databases: Array -): Promise { - const client: DatabaseClient = dbConfig.getString('client'); +// return ( +// ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ?? +// knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)) +// ); +// } - return await ddlLimiter(() => - ConnectorMapping[client]?.ensureDatabaseExists?.(dbConfig, ...databases), - ); -} +// /** +// * Ensures that the given databases all exist, creating them if they do not. +// * +// * @public +// */ +// export async function ensureDatabaseExists( +// dbConfig: Config, +// ...databases: Array +// ): Promise { +// const client: DatabaseClient = dbConfig.getString('client'); -/** - * Drops the given databases. - * - * @public - */ -export async function dropDatabase( - dbConfig: Config, - ...databases: Array -): Promise { - const client: DatabaseClient = dbConfig.getString('client'); +// return await ddlLimiter(() => +// ConnectorMapping[client]?.ensureDatabaseExists?.(dbConfig, ...databases), +// ); +// } - return await ddlLimiter(() => - ConnectorMapping[client]?.dropDatabase?.(dbConfig, ...databases), - ); -} +// /** +// * Drops the given databases. +// * +// * @public +// */ +// export async function dropDatabase( +// dbConfig: Config, +// ...databases: Array +// ): Promise { +// const client: DatabaseClient = dbConfig.getString('client'); -/** - * Ensures that the given schemas all exist, creating them if they do not. - * - * @public - */ -export async function ensureSchemaExists( - dbConfig: Config, - ...schemas: Array -): Promise { - const client: DatabaseClient = dbConfig.getString('client'); +// return await ddlLimiter(() => +// ConnectorMapping[client]?.dropDatabase?.(dbConfig, ...databases), +// ); +// } - return await ddlLimiter(() => - ConnectorMapping[client]?.ensureSchemaExists?.(dbConfig, ...schemas), - ); -} +// /** +// * Ensures that the given schemas all exist, creating them if they do not. +// * +// * @public +// */ +// export async function ensureSchemaExists( +// dbConfig: Config, +// ...schemas: Array +// ): Promise { +// const client: DatabaseClient = dbConfig.getString('client'); -/** - * Provides a `Knex.Config` object with the provided database name for a given - * client. - */ -export function createNameOverride( - client: string, - name: string, -): Partial { - try { - return ConnectorMapping[client].createNameOverride(name); - } catch (e) { - throw new InputError( - `Unable to create database name override for '${client}' connector`, - e, - ); - } -} +// return await ddlLimiter(() => +// ConnectorMapping[client]?.ensureSchemaExists?.(dbConfig, ...schemas), +// ); +// } -/** - * Provides a `Knex.Config` object with the provided database schema for a given - * client. Currently only supported by `pg`. - */ -export function createSchemaOverride( - client: string, - name: string, -): Partial { - try { - return ConnectorMapping[client]?.createSchemaOverride?.(name); - } catch (e) { - throw new InputError( - `Unable to create database schema override for '${client}' connector`, - e, - ); - } -} +// /** +// * Provides a `Knex.Config` object with the provided database name for a given +// * client. +// */ +// export function createNameOverride( +// client: string, +// name: string, +// ): Partial { +// try { +// return ConnectorMapping[client].createNameOverride(name); +// } catch (e) { +// throw new InputError( +// `Unable to create database name override for '${client}' connector`, +// e, +// ); +// } +// } -/** - * Parses a connection string for a given client and provides a connection config. - */ -export function parseConnectionString( - connectionString: string, - client?: string, -): Knex.StaticConnectionConfig { - if (typeof client === 'undefined' || client === null) { - throw new InputError( - 'Database connection string client type auto-detection is not yet supported.', - ); - } +// /** +// * Provides a `Knex.Config` object with the provided database schema for a given +// * client. Currently only supported by `pg`. +// */ +// export function createSchemaOverride( +// client: string, +// name: string, +// ): Partial { +// try { +// return ConnectorMapping[client]?.createSchemaOverride?.(name); +// } catch (e) { +// throw new InputError( +// `Unable to create database schema override for '${client}' connector`, +// e, +// ); +// } +// } - try { - return ConnectorMapping[client].parseConnectionString(connectionString); - } catch (e) { - throw new InputError( - `Unable to parse connection string for '${client}' connector`, - ); - } -} +// /** +// * Parses a connection string for a given client and provides a connection config. +// */ +// export function parseConnectionString( +// connectionString: string, +// client?: string, +// ): Knex.StaticConnectionConfig { +// if (typeof client === 'undefined' || client === null) { +// throw new InputError( +// 'Database connection string client type auto-detection is not yet supported.', +// ); +// } -/** - * Normalizes a connection config or string into an object which can be passed - * to Knex. - */ -export function normalizeConnection( - connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, - client: string, -): Partial { - if (typeof connection === 'undefined' || connection === null) { - return {}; - } +// try { +// return ConnectorMapping[client].parseConnectionString(connectionString); +// } catch (e) { +// throw new InputError( +// `Unable to parse connection string for '${client}' connector`, +// ); +// } +// } - return typeof connection === 'string' || connection instanceof String - ? parseConnectionString(connection as string, client) - : connection; -} +// /** +// * Normalizes a connection config or string into an object which can be passed +// * to Knex. +// */ +// export function normalizeConnection( +// connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, +// client: string, +// ): Partial { +// if (typeof connection === 'undefined' || connection === null) { +// return {}; +// } + +// return typeof connection === 'string' || connection instanceof String +// ? parseConnectionString(connection as string, client) +// : connection; +// } diff --git a/packages/backend-common/src/database/connectors/mysql.ts b/packages/backend-common/src/database/connectors/mysql.ts index a4bac3f7e1..63464fa6f4 100644 --- a/packages/backend-common/src/database/connectors/mysql.ts +++ b/packages/backend-common/src/database/connectors/mysql.ts @@ -14,11 +14,17 @@ * limitations under the License. */ -import knexFactory, { Knex } from 'knex'; -import yn from 'yn'; - -import { Config } from '@backstage/config'; +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; +import { Config, ConfigReader } from '@backstage/config'; import { InputError } from '@backstage/errors'; +import { JsonObject } from '@backstage/types'; +import knexFactory, { Knex } from 'knex'; +import { merge, omit } from 'lodash'; +import path from 'path'; +import yn from 'yn'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; @@ -229,3 +235,325 @@ export const mysqlConnector: DatabaseConnector = Object.freeze({ parseConnectionString: parseMysqlConnectionString, dropDatabase: dropMysqlDatabase, }); + +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Provides a config lookup path for a plugin's config block. + */ +function pluginPath(pluginId: string): string { + return `plugin.${pluginId}`; +} + +function normalizeConnection( + connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, + client: string, +): Partial { + if (typeof connection === 'undefined' || connection === null) { + return {}; + } + + return typeof connection === 'string' || connection instanceof String + ? mysqlConnector.parseConnectionString(connection as string, client) + : connection; +} + +function createSchemaOverride( + client: string, + name: string, +): Partial { + try { + return mysqlConnector.createSchemaOverride?.(name); + } catch (e) { + throw new InputError( + `Unable to create database schema override for '${client}' connector`, + e, + ); + } +} + +function createNameOverride( + client: string, + name: string, +): Partial { + try { + return mysqlConnector.createNameOverride(name); + } catch (e) { + throw new InputError( + `Unable to create database name override for '${client}' connector`, + e, + ); + } +} + +export class MysqlConnector { + constructor( + private readonly config: Config, + private readonly prefix: string = 'backstage_plugin_', + ) {} + + async getClient( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise { + const pluginConfig = new ConfigReader( + this.getConfigForPlugin(pluginId) as JsonObject, + ); + + const databaseName = this.getDatabaseName(pluginId); + if (databaseName && this.getEnsureExistsConfig(pluginId)) { + try { + await mysqlConnector.ensureDatabaseExists(pluginConfig, databaseName); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`, + ); + } + } + + let schemaOverrides; + if (this.getPluginDivisionModeConfig() === 'schema') { + schemaOverrides = this.getSchemaOverrides(pluginId); + if (this.getEnsureExistsConfig(pluginId)) { + try { + await mysqlConnector.ensureSchemaExists(pluginConfig, pluginId); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`, + ); + } + } + } + + const databaseClientOverrides = mergeDatabaseConfig( + {}, + this.getDatabaseOverrides(pluginId), + schemaOverrides, + ); + + const client = mysqlConnector.createClient( + pluginConfig, + databaseClientOverrides, + deps, + ); + + return client; + } + + /** + * Provides the canonical database name for a given plugin. + * + * This method provides the effective database name which is determined using global + * and plugin specific database config. If no explicit database name is configured + * and `pluginDivisionMode` is not `schema`, this method will provide a generated name + * which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is + * `schema`, it will fallback to using the default database for the knex instance. + * + * @param pluginId - Lookup the database name for given plugin + * @returns String representing the plugin's database name + */ + private getDatabaseName(pluginId: string): string | undefined { + const connection = this.getConnectionConfig(pluginId); + + if (this.getClientType(pluginId).client.includes('sqlite3')) { + const sqliteFilename: string | undefined = ( + connection as Knex.Sqlite3ConnectionConfig + ).filename; + + if (sqliteFilename === ':memory:') { + return sqliteFilename; + } + + const sqliteDirectory = + (connection as { directory?: string }).directory ?? '.'; + + return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`); + } + + const databaseName = (connection as Knex.ConnectionConfig)?.database; + + // `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database + if (this.getPluginDivisionModeConfig() === 'schema') { + return databaseName; + } + + // all other supported databases should fallback to an auto-prefixed name + return databaseName ?? `${this.prefix}${pluginId}`; + } + + /** + * Provides the client type which should be used for a given plugin. + * + * The client type is determined by plugin specific config if present. + * Otherwise the base client is used as the fallback. + * + * @param pluginId - Plugin to get the client type for + * @returns Object with client type returned as `client` and boolean + * representing whether or not the client was overridden as + * `overridden` + */ + private getClientType(pluginId: string): { + client: string; + overridden: boolean; + } { + const pluginClient = this.config.getOptionalString( + `${pluginPath(pluginId)}.client`, + ); + + const baseClient = this.config.getString('client'); + const client = pluginClient ?? baseClient; + return { + client, + overridden: client !== baseClient, + }; + } + + private getRoleConfig(pluginId: string): string | undefined { + return ( + this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ?? + this.config.getOptionalString('role') + ); + } + + /** + * Provides the knexConfig which should be used for a given plugin. + * + * @param pluginId - Plugin to get the knexConfig for + * @returns The merged knexConfig value or undefined if it isn't specified + */ + private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined { + const pluginConfig = this.config + .getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`) + ?.get(); + + const baseConfig = this.config + .getOptionalConfig('knexConfig') + ?.get(); + + return merge(baseConfig, pluginConfig); + } + + private getEnsureExistsConfig(pluginId: string): boolean { + const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true; + return ( + this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ?? + baseConfig + ); + } + + private getPluginDivisionModeConfig(): string { + return this.config.getOptionalString('pluginDivisionMode') ?? 'database'; + } + + /** + * Provides a Knex connection plugin config by combining base and plugin + * config. + * + * This method provides a baseConfig for a plugin database connector. If the + * client type has not been overridden, the global connection config will be + * included with plugin specific config as the base. Values from the plugin + * connection take precedence over the base. Base database name is omitted for + * all supported databases excluding SQLite unless `pluginDivisionMode` is set + * to `schema`. + */ + private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig { + const { client, overridden } = this.getClientType(pluginId); + + let baseConnection = normalizeConnection( + this.config.get('connection'), + this.config.getString('client'), + ); + + if ( + client.includes('sqlite3') && + 'filename' in baseConnection && + baseConnection.filename !== ':memory:' + ) { + throw new Error( + '`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.', + ); + } + + // Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The + // `database` property from the base connection is omitted unless `pluginDivisionMode` + // is set to `schema`. SQLite3's `filename` property is an exception as this is used as a + // directory elsewhere so we preserve `filename`. + if (this.getPluginDivisionModeConfig() !== 'schema') { + baseConnection = omit(baseConnection, 'database'); + } + + // get and normalize optional plugin specific database connection + const connection = normalizeConnection( + this.config.getOptional(`${pluginPath(pluginId)}.connection`), + client, + ); + + if (client === 'pg') { + ( + baseConnection as Knex.PgConnectionConfig + ).application_name ||= `backstage_plugin_${pluginId}`; + } + + return { + // include base connection if client type has not been overridden + ...(overridden ? {} : baseConnection), + ...connection, + } as Knex.StaticConnectionConfig; + } + + /** + * Provides a Knex database config for a given plugin. + * + * This method provides a Knex configuration object along with the plugin's + * client type. + * + * @param pluginId - The plugin that the database config should correspond with + */ + private getConfigForPlugin(pluginId: string): Knex.Config { + const { client } = this.getClientType(pluginId); + const role = this.getRoleConfig(pluginId); + + return { + ...this.getAdditionalKnexConfig(pluginId), + client, + connection: this.getConnectionConfig(pluginId), + ...(role && { role }), + }; + } + + /** + * Provides a partial `Knex.Config` database schema override for a given + * plugin. + * + * @param pluginId - Target plugin to get database schema override + * @returns Partial `Knex.Config` with database schema override + */ + private getSchemaOverrides(pluginId: string): Knex.Config | undefined { + return createSchemaOverride(this.getClientType(pluginId).client, pluginId); + } + + /** + * Provides a partial `Knex.Config`• database name override for a given plugin. + * + * @param pluginId - Target plugin to get database name override + * @returns Partial `Knex.Config` with database name override + */ + private getDatabaseOverrides(pluginId: string): Knex.Config { + const databaseName = this.getDatabaseName(pluginId); + return databaseName + ? createNameOverride(this.getClientType(pluginId).client, databaseName) + : {}; + } +} diff --git a/packages/backend-common/src/database/connectors/postgres.ts b/packages/backend-common/src/database/connectors/postgres.ts index dc9b19f6ad..0a3aa6cb74 100644 --- a/packages/backend-common/src/database/connectors/postgres.ts +++ b/packages/backend-common/src/database/connectors/postgres.ts @@ -14,15 +14,21 @@ * limitations under the License. */ +import { + LifecycleService, + PluginMetadataService, +} from '@backstage/backend-plugin-api'; +import { Config, ConfigReader } from '@backstage/config'; +import { ForwardedError, InputError } from '@backstage/errors'; +import { JsonObject } from '@backstage/types'; import knexFactory, { Knex } from 'knex'; - -import { Config } from '@backstage/config'; -import { ForwardedError } from '@backstage/errors'; +import { merge, omit } from 'lodash'; +import path from 'path'; +import { Client } from 'pg'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; import defaultNameOverride from './defaultNameOverride'; import defaultSchemaOverride from './defaultSchemaOverride'; -import { Client } from 'pg'; /** * Creates a knex postgres database connection @@ -217,11 +223,6 @@ export async function dropPgDatabase( ); } -/** - * PostgreSQL database connector. - * - * Exposes database connector functionality via an immutable object. - */ export const pgConnector: DatabaseConnector = Object.freeze({ createClient: createPgDatabaseClient, ensureDatabaseExists: ensurePgDatabaseExists, @@ -231,3 +232,325 @@ export const pgConnector: DatabaseConnector = Object.freeze({ parseConnectionString: parsePgConnectionString, dropDatabase: dropPgDatabase, }); + +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Provides a config lookup path for a plugin's config block. + */ +function pluginPath(pluginId: string): string { + return `plugin.${pluginId}`; +} + +function normalizeConnection( + connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, + client: string, +): Partial { + if (typeof connection === 'undefined' || connection === null) { + return {}; + } + + return typeof connection === 'string' || connection instanceof String + ? pgConnector.parseConnectionString(connection as string, client) + : connection; +} + +function createSchemaOverride( + client: string, + name: string, +): Partial { + try { + return pgConnector.createSchemaOverride?.(name); + } catch (e) { + throw new InputError( + `Unable to create database schema override for '${client}' connector`, + e, + ); + } +} + +function createNameOverride( + client: string, + name: string, +): Partial { + try { + return pgConnector.createNameOverride(name); + } catch (e) { + throw new InputError( + `Unable to create database name override for '${client}' connector`, + e, + ); + } +} + +export class PgConnector { + constructor( + private readonly config: Config, + private readonly prefix: string = 'backstage_plugin_', + ) {} + + async getClient( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise { + const pluginConfig = new ConfigReader( + this.getConfigForPlugin(pluginId) as JsonObject, + ); + + const databaseName = this.getDatabaseName(pluginId); + if (databaseName && this.getEnsureExistsConfig(pluginId)) { + try { + await pgConnector.ensureDatabaseExists(pluginConfig, databaseName); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`, + ); + } + } + + let schemaOverrides; + if (this.getPluginDivisionModeConfig() === 'schema') { + schemaOverrides = this.getSchemaOverrides(pluginId); + if (this.getEnsureExistsConfig(pluginId)) { + try { + await pgConnector.ensureSchemaExists(pluginConfig, pluginId); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`, + ); + } + } + } + + const databaseClientOverrides = mergeDatabaseConfig( + {}, + this.getDatabaseOverrides(pluginId), + schemaOverrides, + ); + + const client = pgConnector.createClient( + pluginConfig, + databaseClientOverrides, + deps, + ); + + return client; + } + + /** + * Provides the canonical database name for a given plugin. + * + * This method provides the effective database name which is determined using global + * and plugin specific database config. If no explicit database name is configured + * and `pluginDivisionMode` is not `schema`, this method will provide a generated name + * which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is + * `schema`, it will fallback to using the default database for the knex instance. + * + * @param pluginId - Lookup the database name for given plugin + * @returns String representing the plugin's database name + */ + private getDatabaseName(pluginId: string): string | undefined { + const connection = this.getConnectionConfig(pluginId); + + if (this.getClientType(pluginId).client.includes('sqlite3')) { + const sqliteFilename: string | undefined = ( + connection as Knex.Sqlite3ConnectionConfig + ).filename; + + if (sqliteFilename === ':memory:') { + return sqliteFilename; + } + + const sqliteDirectory = + (connection as { directory?: string }).directory ?? '.'; + + return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`); + } + + const databaseName = (connection as Knex.ConnectionConfig)?.database; + + // `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database + if (this.getPluginDivisionModeConfig() === 'schema') { + return databaseName; + } + + // all other supported databases should fallback to an auto-prefixed name + return databaseName ?? `${this.prefix}${pluginId}`; + } + + /** + * Provides the client type which should be used for a given plugin. + * + * The client type is determined by plugin specific config if present. + * Otherwise the base client is used as the fallback. + * + * @param pluginId - Plugin to get the client type for + * @returns Object with client type returned as `client` and boolean + * representing whether or not the client was overridden as + * `overridden` + */ + private getClientType(pluginId: string): { + client: string; + overridden: boolean; + } { + const pluginClient = this.config.getOptionalString( + `${pluginPath(pluginId)}.client`, + ); + + const baseClient = this.config.getString('client'); + const client = pluginClient ?? baseClient; + return { + client, + overridden: client !== baseClient, + }; + } + + private getRoleConfig(pluginId: string): string | undefined { + return ( + this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ?? + this.config.getOptionalString('role') + ); + } + + /** + * Provides the knexConfig which should be used for a given plugin. + * + * @param pluginId - Plugin to get the knexConfig for + * @returns The merged knexConfig value or undefined if it isn't specified + */ + private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined { + const pluginConfig = this.config + .getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`) + ?.get(); + + const baseConfig = this.config + .getOptionalConfig('knexConfig') + ?.get(); + + return merge(baseConfig, pluginConfig); + } + + private getEnsureExistsConfig(pluginId: string): boolean { + const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true; + return ( + this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ?? + baseConfig + ); + } + + private getPluginDivisionModeConfig(): string { + return this.config.getOptionalString('pluginDivisionMode') ?? 'database'; + } + + /** + * Provides a Knex connection plugin config by combining base and plugin + * config. + * + * This method provides a baseConfig for a plugin database connector. If the + * client type has not been overridden, the global connection config will be + * included with plugin specific config as the base. Values from the plugin + * connection take precedence over the base. Base database name is omitted for + * all supported databases excluding SQLite unless `pluginDivisionMode` is set + * to `schema`. + */ + private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig { + const { client, overridden } = this.getClientType(pluginId); + + let baseConnection = normalizeConnection( + this.config.get('connection'), + this.config.getString('client'), + ); + + if ( + client.includes('sqlite3') && + 'filename' in baseConnection && + baseConnection.filename !== ':memory:' + ) { + throw new Error( + '`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.', + ); + } + + // Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The + // `database` property from the base connection is omitted unless `pluginDivisionMode` + // is set to `schema`. SQLite3's `filename` property is an exception as this is used as a + // directory elsewhere so we preserve `filename`. + if (this.getPluginDivisionModeConfig() !== 'schema') { + baseConnection = omit(baseConnection, 'database'); + } + + // get and normalize optional plugin specific database connection + const connection = normalizeConnection( + this.config.getOptional(`${pluginPath(pluginId)}.connection`), + client, + ); + + if (client === 'pg') { + ( + baseConnection as Knex.PgConnectionConfig + ).application_name ||= `backstage_plugin_${pluginId}`; + } + + return { + // include base connection if client type has not been overridden + ...(overridden ? {} : baseConnection), + ...connection, + } as Knex.StaticConnectionConfig; + } + + /** + * Provides a Knex database config for a given plugin. + * + * This method provides a Knex configuration object along with the plugin's + * client type. + * + * @param pluginId - The plugin that the database config should correspond with + */ + private getConfigForPlugin(pluginId: string): Knex.Config { + const { client } = this.getClientType(pluginId); + const role = this.getRoleConfig(pluginId); + + return { + ...this.getAdditionalKnexConfig(pluginId), + client, + connection: this.getConnectionConfig(pluginId), + ...(role && { role }), + }; + } + + /** + * Provides a partial `Knex.Config` database schema override for a given + * plugin. + * + * @param pluginId - Target plugin to get database schema override + * @returns Partial `Knex.Config` with database schema override + */ + private getSchemaOverrides(pluginId: string): Knex.Config | undefined { + return createSchemaOverride(this.getClientType(pluginId).client, pluginId); + } + + /** + * Provides a partial `Knex.Config`• database name override for a given plugin. + * + * @param pluginId - Target plugin to get database name override + * @returns Partial `Knex.Config` with database name override + */ + private getDatabaseOverrides(pluginId: string): Knex.Config { + const databaseName = this.getDatabaseName(pluginId); + return databaseName + ? createNameOverride(this.getClientType(pluginId).client, databaseName) + : {}; + } +} diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 3cb0f57de6..5714b036e3 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -14,17 +14,20 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import { ensureDirSync } from 'fs-extra'; -import knexFactory, { Knex } from 'knex'; -import path from 'path'; import { DevDataStore } from '@backstage/backend-dev-utils'; -import { mergeDatabaseConfig } from '../config'; -import { DatabaseConnector } from '../types'; import { LifecycleService, PluginMetadataService, } from '@backstage/backend-plugin-api'; +import { Config, ConfigReader } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { JsonObject } from '@backstage/types'; +import { ensureDirSync } from 'fs-extra'; +import knexFactory, { Knex } from 'knex'; +import { merge, omit } from 'lodash'; +import path from 'path'; +import { mergeDatabaseConfig } from '../config'; +import { DatabaseConnector } from '../types'; /** * Creates a knex SQLite3 database connection @@ -159,8 +162,330 @@ export function parseSqliteConnectionString( * * Exposes database connector functionality via an immutable object. */ -export const sqlite3Connector: DatabaseConnector = Object.freeze({ +export const sqliteConnector: DatabaseConnector = Object.freeze({ createClient: createSqliteDatabaseClient, createNameOverride: createSqliteNameOverride, parseConnectionString: parseSqliteConnectionString, }); + +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Provides a config lookup path for a plugin's config block. + */ +function pluginPath(pluginId: string): string { + return `plugin.${pluginId}`; +} + +function normalizeConnection( + connection: Knex.StaticConnectionConfig | JsonObject | string | undefined, + client: string, +): Partial { + if (typeof connection === 'undefined' || connection === null) { + return {}; + } + + return typeof connection === 'string' || connection instanceof String + ? sqliteConnector.parseConnectionString(connection as string, client) + : connection; +} + +function createSchemaOverride( + client: string, + name: string, +): Partial { + try { + return sqliteConnector.createSchemaOverride?.(name); + } catch (e) { + throw new InputError( + `Unable to create database schema override for '${client}' connector`, + e, + ); + } +} + +function createNameOverride( + client: string, + name: string, +): Partial { + try { + return sqliteConnector.createNameOverride(name); + } catch (e) { + throw new InputError( + `Unable to create database name override for '${client}' connector`, + e, + ); + } +} + +export class Sqlite3Connector implements DatabaseConnector { + constructor( + private readonly config: Config, + private readonly prefix: string = 'backstage_plugin_', + ) {} + + async getClient( + pluginId: string, + deps?: { + lifecycle: LifecycleService; + pluginMetadata: PluginMetadataService; + }, + ): Promise { + const pluginConfig = new ConfigReader( + this.getConfigForPlugin(pluginId) as JsonObject, + ); + + const databaseName = this.getDatabaseName(pluginId); + if (databaseName && this.getEnsureExistsConfig(pluginId)) { + try { + await sqliteConnector.ensureDatabaseExists(pluginConfig, databaseName); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`, + ); + } + } + + let schemaOverrides; + if (this.getPluginDivisionModeConfig() === 'schema') { + schemaOverrides = this.getSchemaOverrides(pluginId); + if (this.getEnsureExistsConfig(pluginId)) { + try { + await sqliteConnector.ensureSchemaExists(pluginConfig, pluginId); + } catch (error) { + throw new Error( + `Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`, + ); + } + } + } + + const databaseClientOverrides = mergeDatabaseConfig( + {}, + this.getDatabaseOverrides(pluginId), + schemaOverrides, + ); + + const client = sqliteConnector.createClient( + pluginConfig, + databaseClientOverrides, + deps, + ); + + return client; + } + + /** + * Provides the canonical database name for a given plugin. + * + * This method provides the effective database name which is determined using global + * and plugin specific database config. If no explicit database name is configured + * and `pluginDivisionMode` is not `schema`, this method will provide a generated name + * which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is + * `schema`, it will fallback to using the default database for the knex instance. + * + * @param pluginId - Lookup the database name for given plugin + * @returns String representing the plugin's database name + */ + private getDatabaseName(pluginId: string): string | undefined { + const connection = this.getConnectionConfig(pluginId); + + if (this.getClientType(pluginId).client.includes('sqlite3')) { + const sqliteFilename: string | undefined = ( + connection as Knex.Sqlite3ConnectionConfig + ).filename; + + if (sqliteFilename === ':memory:') { + return sqliteFilename; + } + + const sqliteDirectory = + (connection as { directory?: string }).directory ?? '.'; + + return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`); + } + + const databaseName = (connection as Knex.ConnectionConfig)?.database; + + // `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database + if (this.getPluginDivisionModeConfig() === 'schema') { + return databaseName; + } + + // all other supported databases should fallback to an auto-prefixed name + return databaseName ?? `${this.prefix}${pluginId}`; + } + + /** + * Provides the client type which should be used for a given plugin. + * + * The client type is determined by plugin specific config if present. + * Otherwise the base client is used as the fallback. + * + * @param pluginId - Plugin to get the client type for + * @returns Object with client type returned as `client` and boolean + * representing whether or not the client was overridden as + * `overridden` + */ + private getClientType(pluginId: string): { + client: string; + overridden: boolean; + } { + const pluginClient = this.config.getOptionalString( + `${pluginPath(pluginId)}.client`, + ); + + const baseClient = this.config.getString('client'); + const client = pluginClient ?? baseClient; + return { + client, + overridden: client !== baseClient, + }; + } + + private getRoleConfig(pluginId: string): string | undefined { + return ( + this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ?? + this.config.getOptionalString('role') + ); + } + + /** + * Provides the knexConfig which should be used for a given plugin. + * + * @param pluginId - Plugin to get the knexConfig for + * @returns The merged knexConfig value or undefined if it isn't specified + */ + private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined { + const pluginConfig = this.config + .getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`) + ?.get(); + + const baseConfig = this.config + .getOptionalConfig('knexConfig') + ?.get(); + + return merge(baseConfig, pluginConfig); + } + + private getEnsureExistsConfig(pluginId: string): boolean { + const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true; + return ( + this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ?? + baseConfig + ); + } + + private getPluginDivisionModeConfig(): string { + return this.config.getOptionalString('pluginDivisionMode') ?? 'database'; + } + + /** + * Provides a Knex connection plugin config by combining base and plugin + * config. + * + * This method provides a baseConfig for a plugin database connector. If the + * client type has not been overridden, the global connection config will be + * included with plugin specific config as the base. Values from the plugin + * connection take precedence over the base. Base database name is omitted for + * all supported databases excluding SQLite unless `pluginDivisionMode` is set + * to `schema`. + */ + private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig { + const { client, overridden } = this.getClientType(pluginId); + + let baseConnection = normalizeConnection( + this.config.get('connection'), + this.config.getString('client'), + ); + + if ( + client.includes('sqlite3') && + 'filename' in baseConnection && + baseConnection.filename !== ':memory:' + ) { + throw new Error( + '`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.', + ); + } + + // Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The + // `database` property from the base connection is omitted unless `pluginDivisionMode` + // is set to `schema`. SQLite3's `filename` property is an exception as this is used as a + // directory elsewhere so we preserve `filename`. + if (this.getPluginDivisionModeConfig() !== 'schema') { + baseConnection = omit(baseConnection, 'database'); + } + + // get and normalize optional plugin specific database connection + const connection = normalizeConnection( + this.config.getOptional(`${pluginPath(pluginId)}.connection`), + client, + ); + + if (client === 'pg') { + ( + baseConnection as Knex.PgConnectionConfig + ).application_name ||= `backstage_plugin_${pluginId}`; + } + + return { + // include base connection if client type has not been overridden + ...(overridden ? {} : baseConnection), + ...connection, + } as Knex.StaticConnectionConfig; + } + + /** + * Provides a Knex database config for a given plugin. + * + * This method provides a Knex configuration object along with the plugin's + * client type. + * + * @param pluginId - The plugin that the database config should correspond with + */ + private getConfigForPlugin(pluginId: string): Knex.Config { + const { client } = this.getClientType(pluginId); + const role = this.getRoleConfig(pluginId); + + return { + ...this.getAdditionalKnexConfig(pluginId), + client, + connection: this.getConnectionConfig(pluginId), + ...(role && { role }), + }; + } + + /** + * Provides a partial `Knex.Config` database schema override for a given + * plugin. + * + * @param pluginId - Target plugin to get database schema override + * @returns Partial `Knex.Config` with database schema override + */ + private getSchemaOverrides(pluginId: string): Knex.Config | undefined { + return createSchemaOverride(this.getClientType(pluginId).client, pluginId); + } + + /** + * Provides a partial `Knex.Config`• database name override for a given plugin. + * + * @param pluginId - Target plugin to get database name override + * @returns Partial `Knex.Config` with database name override + */ + private getDatabaseOverrides(pluginId: string): Knex.Config { + const databaseName = this.getDatabaseName(pluginId); + return databaseName + ? createNameOverride(this.getClientType(pluginId).client, databaseName) + : {}; + } +} diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts index 3e9832aec5..c73540a9c1 100644 --- a/packages/backend-common/src/database/types.ts +++ b/packages/backend-common/src/database/types.ts @@ -24,11 +24,11 @@ import { Knex } from 'knex'; export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api'; /** - * DatabaseConnector manages an underlying Knex database driver. + * Manages an underlying Knex database driver. */ export interface DatabaseConnector { /** - * createClient provides an instance of a knex database connector. + * Provides an instance of a knex database connector. */ createClient( dbConfig: Config, @@ -38,27 +38,29 @@ export interface DatabaseConnector { pluginMetadata: PluginMetadataService; }, ): Knex; + /** - * createNameOverride provides a partial knex config sufficient to override a - * database name. + * Provides a partial knex config sufficient to override a database name. */ createNameOverride(name: string): Partial; + /** - * createSchemaOverride provides a partial knex config sufficient to override a - * PostgreSQL schema name within utilizing the `searchPath` knex configuration. + * Provides a partial knex config sufficient to override a PostgreSQL schema + * name within utilizing the `searchPath` knex configuration. */ createSchemaOverride?(name: string): Partial; + /** - * parseConnectionString produces a knex connection config object representing - * a database connection string. + * Produces a knex connection config object representing a database connection + * string. */ parseConnectionString( connectionString: string, client?: string, ): Knex.StaticConnectionConfig; + /** - * ensureDatabaseExists performs a side-effect to ensure database names passed in are - * present. + * Performs a side-effect to ensure database names passed in are present. * * Calling this function on databases which already exist should do nothing. * Missing databases should be created if needed. @@ -69,8 +71,7 @@ export interface DatabaseConnector { ): Promise; /** - * ensureSchemaExists performs a side-effect to ensure schema names passed in are - * present. + * Performs a side-effect to ensure schema names passed in are present. * * Calling this function on schemas which already exist should do nothing. * Missing schemas should be created if needed. @@ -80,5 +81,8 @@ export interface DatabaseConnector { ...schemas: Array ): Promise; + /** + * Deletes databases. + */ dropDatabase?(dbConfig: Config, ...databases: Array): Promise; }