paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

slim down the origin database manager

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2024-04-28 11:47:53 +02:00
parent 1da71d800c
commit c936bfc67c
15 changed files with 50 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/src/database/DatabaseManager.test.ts
+1 -1
View File
@@ -16,12 +16,12 @@
import { ConfigReader } from '@backstage/config';
import { omit } from 'lodash';
import path from 'path';
import { DatabaseManager } from './DatabaseManager';
import {
createDatabaseClient,
ensureDatabaseExists,
ensureSchemaExists,
} from './connection';
import { DatabaseManager } from './DatabaseManager';
jest.mock('./connection', () => ({
...jest.requireActual('./connection'),
packages/backend-common/src/database/DatabaseManager.ts
+21 -254
View File
@@ -14,23 +14,19 @@
* limitations under the License.
*/
import { Config, ConfigReader } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { Knex } from 'knex';
import { merge, omit } from 'lodash';
import { mergeDatabaseConfig } from './config';
import { PluginDatabaseManager } from './types';
import path from 'path';
import {
DatabaseService,
LifecycleService,
LoggerService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config } from '@backstage/config';
import { stringifyError } from '@backstage/errors';
import { Knex } from 'knex';
import { MysqlConnector } from './connectors/mysql';
import { PgConnector } from './connectors/postgres';
import { Sqlite3Connector } from './connectors/sqlite3';
import { MysqlConnector } from './connectors/mysql';
import { Connector, PluginDatabaseManager } from './types';
/**
* Provides a config lookup path for a plugin's config block.
@@ -81,15 +77,15 @@ export class DatabaseManager implements LegacyRootDatabaseService {
options?: DatabaseManagerOptions,
): DatabaseManager {
const databaseConfig = config.getConfig('backend.database');
const prefix = databaseConfig.getOptionalString('prefix');
return new DatabaseManager(
databaseConfig,
databaseConfig.getOptionalString('prefix'),
{
pg: new PgConnector(config, prefix, options),
sqlite3: new Sqlite3Connector(config, prefix, options),
'better-sqlite3': new Sqlite3Connector(config, prefix, options),
mysql: new MysqlConnector(config, prefix, options),
mysql2: new MysqlConnector(config, prefix, options),
pg: new PgConnector(config, prefix),
sqlite3: new Sqlite3Connector(config, prefix),
'better-sqlite3': new Sqlite3Connector(config, prefix),
mysql: new MysqlConnector(config, prefix),
mysql2: new MysqlConnector(config, prefix),
},
options,
);
@@ -97,19 +93,7 @@ export class DatabaseManager implements LegacyRootDatabaseService {
private constructor(
private readonly config: Config,
private readonly prefix: string = 'backstage_plugin_',
private readonly connectors: Record<
string,
{
getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex>;
}
>,
private readonly connectors: Record<string, Connector>,
private readonly options?: DatabaseManagerOptions,
private readonly databaseCache: Map<string, Promise<Knex>> = new Map(),
) {}
@@ -131,54 +115,15 @@ export class DatabaseManager implements LegacyRootDatabaseService {
const client = this.getClientType(pluginId).client;
const connector = this.connectors[client];
if (!connector) {
throw new Error(`Unsupported database client type '${client}'`);
throw new Error(
`Unsupported database client type '${client}' specified for plugin '${pluginId}'`,
);
}
const getClient = () => this.getDatabase(pluginId, deps);
const getClient = () => this.getDatabase(pluginId, connector, deps);
const migrations = { skip: false, ...this.options?.migrations };
return { getClient, migrations };
}
/**
* Provides the canonical database name for a given plugin.
*
* This method provides the effective database name which is determined using global
* and plugin specific database config. If no explicit database name is configured
* and `pluginDivisionMode` is not `schema`, this method will provide a generated name
* which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is
* `schema`, it will fallback to using the default database for the knex instance.
*
* @param pluginId - Lookup the database name for given plugin
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
if (this.getClientType(pluginId).client.includes('sqlite3')) {
const sqliteFilename: string | undefined = (
connection as Knex.Sqlite3ConnectionConfig
).filename;
if (sqliteFilename === ':memory:') {
return sqliteFilename;
}
const sqliteDirectory =
(connection as { directory?: string }).directory ?? '.';
return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`);
}
const databaseName = (connection as Knex.ConnectionConfig)?.database;
// `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database
if (this.getPluginDivisionModeConfig() === 'schema') {
return databaseName;
}
// all other supported databases should fallback to an auto-prefixed name
return databaseName ?? `${this.prefix}${pluginId}`;
}
/**
* Provides the client type which should be used for a given plugin.
*
@@ -206,143 +151,6 @@ export class DatabaseManager implements LegacyRootDatabaseService {
};
}
private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}
/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();
const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();
return merge(baseConfig, pluginConfig);
}
private getEnsureExistsConfig(pluginId: string): boolean {
const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true;
return (
this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ??
baseConfig
);
}
private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}
/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted for
* all supported databases excluding SQLite unless `pluginDivisionMode` is set
* to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);
let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);
if (
client.includes('sqlite3') &&
'filename' in baseConnection &&
baseConnection.filename !== ':memory:'
) {
throw new Error(
'`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.',
);
}
// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`. SQLite3's `filename` property is an exception as this is used as a
// directory elsewhere so we preserve `filename`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}
// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);
if (client === 'pg') {
(
baseConnection as Knex.PgConnectionConfig
).application_name ||= `backstage_plugin_${pluginId}`;
}
return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}
/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);
return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}
/**
* Provides a partial `Knex.Config` database schema override for a given
* plugin.
*
* @param pluginId - Target plugin to get database schema override
* @returns Partial `Knex.Config` with database schema override
*/
private getSchemaOverrides(pluginId: string): Knex.Config | undefined {
return createSchemaOverride(this.getClientType(pluginId).client, pluginId);
}
/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}
/**
* Provides a scoped Knex client for a plugin as per application config.
*
@@ -352,6 +160,7 @@ export class DatabaseManager implements LegacyRootDatabaseService {
*/
private async getDatabase(
pluginId: string,
connector: Connector,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
@@ -361,55 +170,13 @@ export class DatabaseManager implements LegacyRootDatabaseService {
return this.databaseCache.get(pluginId)!;
}
const clientPromise = Promise.resolve().then(async () => {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
);
const databaseName = this.getDatabaseName(pluginId);
if (databaseName && this.getEnsureExistsConfig(pluginId)) {
try {
await ensureDatabaseExists(pluginConfig, databaseName);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`,
);
}
}
let schemaOverrides;
if (this.getPluginDivisionModeConfig() === 'schema') {
schemaOverrides = this.getSchemaOverrides(pluginId);
if (this.getEnsureExistsConfig(pluginId)) {
try {
await ensureSchemaExists(pluginConfig, pluginId);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`,
);
}
}
}
const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
schemaOverrides,
);
const client = createDatabaseClient(
pluginConfig,
databaseClientOverrides,
deps,
);
if (process.env.NODE_ENV !== 'test') {
this.startKeepaliveLoop(pluginId, client);
}
return client;
});
const clientPromise = connector.getClient(pluginId, deps).then();
this.databaseCache.set(pluginId, clientPromise);
if (process.env.NODE_ENV !== 'test') {
clientPromise.then(client => this.startKeepaliveLoop(pluginId, client));
}
return clientPromise;
}
packages/backend-common/src/database/connectors/defaultNameOverride.test.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import defaultNameOverride from './defaultNameOverride';
describe('defaultNameOverride()', () => {
packages/backend-common/src/database/connectors/defaultNameOverride.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex } from 'knex';
/**
packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import defaultSchemaOverride from './defaultSchemaOverride';
describe('defaultNameOverride()', () => {
packages/backend-common/src/database/connectors/defaultSchemaOverride.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex } from 'knex';
/**
packages/backend-common/src/database/connectors/index.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './mysql';
export * from './postgres';
export * from './sqlite3';
packages/backend-common/src/database/config.test.ts → packages/backend-common/src/database/connectors/mergeDatabaseConfig.test.ts
+2 -2
View File
@@ -14,9 +14,9 @@
* limitations under the License.
*/
import { mergeDatabaseConfig } from './config';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
describe('config', () => {
describe('mergeDatabaseConfig', () => {
describe('mergeDatabaseConfig', () => {
it('does not mutate the input object', () => {
const input = {
packages/backend-common/src/database/config.ts → packages/backend-common/src/database/connectors/mergeDatabaseConfig.ts
View File
packages/backend-common/src/database/connectors/mysql.ts
+3 -3
View File
@@ -25,9 +25,9 @@ import knexFactory, { Knex } from 'knex';
import { merge, omit } from 'lodash';
import path from 'path';
import yn from 'yn';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import { Connector, DatabaseConnector } from '../types';
import defaultNameOverride from './defaultNameOverride';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
/**
* Creates a knex mysql database connection
@@ -295,7 +295,7 @@ function createNameOverride(
}
}
export class MysqlConnector {
export class MysqlConnector implements Connector {
constructor(
private readonly config: Config,
private readonly prefix: string = 'backstage_plugin_',
packages/backend-common/src/database/connectors/postgres.test.ts
+1 -1
View File
@@ -16,8 +16,8 @@
import { Config, ConfigReader } from '@backstage/config';
import {
createPgDatabaseClient,
buildPgDatabaseConfig,
createPgDatabaseClient,
getPgConnectionConfig,
parsePgConnectionString,
} from './postgres';
packages/backend-common/src/database/connectors/postgres.ts
+3 -3
View File
@@ -25,10 +25,10 @@ import knexFactory, { Knex } from 'knex';
import { merge, omit } from 'lodash';
import path from 'path';
import { Client } from 'pg';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import { Connector, DatabaseConnector } from '../types';
import defaultNameOverride from './defaultNameOverride';
import defaultSchemaOverride from './defaultSchemaOverride';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
/**
* Creates a knex postgres database connection
@@ -292,7 +292,7 @@ function createNameOverride(
}
}
export class PgConnector {
export class PgConnector implements Connector {
constructor(
private readonly config: Config,
private readonly prefix: string = 'backstage_plugin_',
packages/backend-common/src/database/connectors/sqlite3.ts
+3 -3
View File
@@ -26,8 +26,8 @@ import { ensureDirSync } from 'fs-extra';
import knexFactory, { Knex } from 'knex';
import { merge, omit } from 'lodash';
import path from 'path';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import { Connector, DatabaseConnector } from '../types';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
/**
* Creates a knex SQLite3 database connection
@@ -227,7 +227,7 @@ function createNameOverride(
}
}
export class Sqlite3Connector implements DatabaseConnector {
export class Sqlite3Connector implements Connector {
constructor(
private readonly config: Config,
private readonly prefix: string = 'backstage_plugin_',
packages/backend-common/src/database/index.ts
+1 -1
View File
@@ -22,8 +22,8 @@ export * from './DatabaseManager';
*/
export {
createDatabaseClient,
ensureDatabaseExists,
dropDatabase,
ensureDatabaseExists,
} from './connection';
export type { PluginDatabaseManager } from './types';
packages/backend-common/src/database/types.ts
+10
View File
@@ -86,3 +86,13 @@ export interface DatabaseConnector {
*/
dropDatabase?(dbConfig: Config, ...databases: Array<string>): Promise<void>;
}
export interface Connector {
getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex>;
}