paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Merge pull request #24545 from backstage/freben/database-refactor

Browse Source 
🧹 rearrange the backend-common database code
This commit is contained in:
Fredrik Adelöw
2024-04-30 12:37:54 +02:00
committed by GitHub
parent 28571a64fa 1cd13b181b
commit 0992c7e9d1
19 changed files with 1061 additions and 1674 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/mean-ravens-dance.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/backend-common': minor
---
Internal refactor of the database code.
**BREAKING**: The helper functions `createDatabaseClient` and `ensureDatabaseExists` have been removed from the public interface, since they have no usage within the repository and never were suitable for calling from the outside. Please consider using `coreServices.database` or `DatabaseManager` directly wherever possible instead.
packages/backend-common/api-report.md
+1 -19
View File
@@ -34,8 +34,6 @@ import { HostDiscovery as HostDiscovery_2 } from '@backstage/backend-app-api';
import { HttpAuthService } from '@backstage/backend-plugin-api';
import { IdentityService } from '@backstage/backend-plugin-api';
import { isChildPath } from '@backstage/cli-common';
import { Knex } from 'knex';
import knexFactory from 'knex';
import { KubeConfig } from '@kubernetes/client-node';
import { LifecycleService } from '@backstage/backend-plugin-api';
import { LoadConfigOptionsRemote } from '@backstage/config-loader';
@@ -225,16 +223,6 @@ export interface ContainerRunner {
runContainer(opts: RunContainerOptions): Promise<void>;
}
// @public
export function createDatabaseClient(
dbConfig: Config,
overrides?: Partial<Knex.Config>,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): knexFactory.Knex<any, any[]>;
// @public
export function createLegacyAuthAdapters<
TOptions extends {
@@ -315,13 +303,7 @@ export class DockerContainerRunner implements ContainerRunner {
// @public
export function dropDatabase(
dbConfig: Config,
...databases: Array<string>
): Promise<void>;
// @public
export function ensureDatabaseExists(
dbConfig: Config,
...databases: Array<string>
...databaseNames: string[]
): Promise<void>;
// @public
packages/backend-common/src/database/DatabaseManager.test.ts
+84 -812
View File
@@ -13,839 +13,111 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { omit } from 'lodash';
import path from 'path';
import {
createDatabaseClient,
ensureDatabaseExists,
ensureSchemaExists,
} from './connection';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseManagerImpl } from './DatabaseManager';
import { Connector } from './types';
jest.mock('./connection', () => ({
...jest.requireActual('./connection'),
createDatabaseClient: jest.fn(),
ensureDatabaseExists: jest.fn(),
ensureSchemaExists: jest.fn(),
}));
describe('DatabaseManager', () => {
// This is similar to the ts-jest `mocked` helper.
const mocked = (f: Function) => f as jest.Mock;
afterEach(() => jest.resetAllMocks());
describe('DatabaseManager.fromConfig', () => {
const backendConfig = {
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
};
it('accesses the backend.database key', () => {
const config = new ConfigReader(backendConfig);
const getConfigSpy = jest.spyOn(config, 'getConfig');
DatabaseManager.fromConfig(config);
expect(getConfigSpy).toHaveBeenCalledWith('backend.database');
});
it('handles default options', () => {
const config = new ConfigReader(backendConfig);
const database = DatabaseManager.fromConfig(config);
const client = database.forPlugin('test');
expect(client.migrations?.skip).toBe(false);
});
it('handles migrations options', () => {
const config = new ConfigReader(backendConfig);
const database = DatabaseManager.fromConfig(config, {
migrations: { skip: true },
});
const client = database.forPlugin('test');
expect(client.migrations?.skip).toBe(true);
});
describe('DatabaseManagerImpl', () => {
afterEach(() => {
jest.clearAllMocks();
});
describe('DatabaseManager.forPlugin', () => {
const config = {
backend: {
database: {
client: 'pg',
prefix: 'test_prefix_',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
plugin: {
testdbname: {
connection: {
database: 'database_name_overridden',
},
},
differentclient: {
client: 'better-sqlite3',
connection: {
filename: 'plugin_with_different_client',
},
},
differentclientconnstring: {
client: 'better-sqlite3',
connection: ':memory:',
},
stringoverride: {
connection: 'postgresql://testuser:testpass@acme:5432/userdbname',
},
},
},
it('calls the right connector, only once per plugin id', async () => {
const connector1 = {
getClient: jest.fn(),
dropDatabase: jest.fn(),
} satisfies Connector;
const connector2 = {
getClient: jest.fn(),
dropDatabase: jest.fn(),
} satisfies Connector;
const impl = new DatabaseManagerImpl(
new ConfigReader({
client: 'pg',
}),
{
pg: connector1,
notpg: connector2,
},
};
let manager: DatabaseManager;
);
beforeEach(() => {
manager = DatabaseManager.fromConfig(new ConfigReader(config));
});
await impl.forPlugin('plugin1').getClient();
expect(connector1.getClient).toHaveBeenCalledTimes(1);
expect(connector1.getClient).toHaveBeenLastCalledWith('plugin1', undefined);
expect(connector2.getClient).toHaveBeenCalledTimes(0);
it('connects to a plugin database using default config', async () => {
const pluginId = 'pluginwithoutconfig';
await impl.forPlugin('plugin1').getClient();
expect(connector1.getClient).toHaveBeenCalledTimes(1);
expect(connector1.getClient).toHaveBeenLastCalledWith('plugin1', undefined);
expect(connector2.getClient).toHaveBeenCalledTimes(0);
await manager.forPlugin(pluginId).getClient();
expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1);
await impl.forPlugin('plugin2').getClient();
expect(connector1.getClient).toHaveBeenCalledTimes(2);
expect(connector1.getClient).toHaveBeenLastCalledWith('plugin2', undefined);
expect(connector2.getClient).toHaveBeenCalledTimes(0);
});
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
it('respects per-plugin overridden connectors', async () => {
const connector1 = {
getClient: jest.fn(),
dropDatabase: jest.fn(),
} satisfies Connector;
const connector2 = {
getClient: jest.fn(),
dropDatabase: jest.fn(),
} satisfies Connector;
// default config should be passed through to underlying connector
expect(baseConfig.get()).toMatchObject({
const impl = new DatabaseManagerImpl(
new ConfigReader({
client: 'pg',
connection: omit(config.backend.database.connection, ['database']),
});
// override using database name generated from pluginId and prefix
expect(overrides).toMatchObject({
connection: {
database: `${config.backend.database.prefix}${pluginId}`,
},
});
});
it('provides a plugin db which uses components from top level connection string', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: 'postgresql://foo:bar@acme:5432/foodb',
},
},
}),
);
await testManager.forPlugin('pluginwithoutconfig').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
// parsed connection string **without** db name should be passed through
expect(baseConfig.get()).toMatchObject({
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
port: '5432',
application_name: 'backstage_plugin_pluginwithoutconfig',
},
});
// we expect a pg database name override with ${prefix} followed by pluginId
expect(overrides).toHaveProperty(
'connection.database',
expect.stringContaining('pluginwithoutconfig'),
);
});
it('provides an inmemory sqlite database if top level is also inmemory and plugin config is not present', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: ':memory:',
},
},
}),
);
await testManager.forPlugin('pluginwithoutconfig').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.filename',
expect.stringContaining(':memory:'),
);
});
it('throws if top level sqlite filename is provided', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: 'some-file-path',
},
},
}),
);
await expect(
testManager.forPlugin('pluginwithoutconfig').getClient(),
).rejects.toBeInstanceOf(Error);
});
it('creates plugin-specific sqlite files when plugin config is not present', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: {
directory: 'sqlite-files',
},
},
},
}),
);
await testManager.forPlugin('pluginwithoutconfig').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.filename',
path.join('sqlite-files', 'pluginwithoutconfig.sqlite'),
);
});
it('uses sqlite directory from top level config and filename from plugin config', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: {
directory: 'sqlite-files',
},
plugin: {
test: {
connection: {
filename: 'other.sqlite',
},
},
},
},
},
}),
);
await testManager.forPlugin('test').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.filename',
path.join('sqlite-files', 'other.sqlite'),
);
});
it('uses sqlite directory and filename from plugin config', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: {
directory: 'sqlite-files',
},
plugin: {
test: {
connection: {
directory: 'custom-sqlite-files',
filename: 'other.sqlite',
},
},
},
},
},
}),
);
await testManager.forPlugin('test').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.filename',
path.join('custom-sqlite-files', 'other.sqlite'),
);
});
it('connects to a plugin database using a specific database name', async () => {
// testdbname.connection.database is set in config
await manager.forPlugin('testdbname').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
// simple case where only database name is overridden
expect(overrides).toMatchObject({
connection: {
database: 'database_name_overridden',
},
});
});
it('ensure plugin specific database is created', async () => {
const pluginId = 'testdbname';
// testdbname.connection.database is set in config
await manager.forPlugin(pluginId).getClient();
const mockCalls = mocked(ensureDatabaseExists).mock.calls.splice(-1);
const [_, dbname] = mockCalls[0];
expect(dbname).toEqual(
config.backend.database.plugin[pluginId].connection.database,
);
});
it('provides different plugins with their own databases', async () => {
await manager.forPlugin('plugin1').getClient();
await manager.forPlugin('plugin2').getClient();
expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(2);
const mockCalls = mocked(createDatabaseClient).mock.calls;
const [plugin1CallArgs, plugin2CallArgs] = mockCalls;
// database name overrides should be different
expect(plugin1CallArgs[1].connection.database).not.toEqual(
plugin2CallArgs[1].connection.database,
);
});
it('returns the same client for the same pluginId', async () => {
const [client1, client2] = await Promise.all([
manager.forPlugin('plugin1').getClient(),
manager.forPlugin('plugin1').getClient(),
]);
expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1);
expect(client1).toBe(client2);
});
it('uses plugin connection as base if default client is different from plugin client', async () => {
const pluginId = 'differentclient';
await manager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, _overrides] = mockCalls[0];
// plugin connection should be used as base config, client is different
expect(baseConfig.get()).toMatchObject({
client: 'better-sqlite3',
connection: config.backend.database.plugin[pluginId].connection,
});
});
it('provides database client specific base and override when client set under plugin', async () => {
const pluginId = 'differentclient';
await manager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
// plugin client should be better-sqlite3
expect(baseConfig.get().client).toEqual('better-sqlite3');
// SQLite uses 'filename' instead of 'database'
expect(overrides).toHaveProperty(
'connection.filename',
'plugin_with_different_client',
);
});
it('provides database client specific base from plugin connection string when client set under plugin', async () => {
const pluginId = 'differentclientconnstring';
await manager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
expect(baseConfig.get().client).toEqual('better-sqlite3');
expect(overrides).toHaveProperty('connection.filename', ':memory:');
});
it('generates a database name override when prefix is not explicitly set', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
await testManager.forPlugin('testplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.database',
expect.stringContaining('backstage_plugin_'),
);
});
it('generates a database name override when prefix is not explicitly set for mysql', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'mysql',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
await testManager.forPlugin('testplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.database',
expect.stringContaining('backstage_plugin_'),
);
});
it('uses values from plugin connection string if top level client should be used', async () => {
const pluginId = 'stringoverride';
await manager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
// plugin client should be pg
expect(baseConfig.get().client).toEqual('pg');
expect(overrides).toHaveProperty(
'connection.database',
expect.stringContaining('userdbname'),
);
});
it('plugin sets schema override for pg client', async () => {
const overrideConfig = {
backend: {
database: {
client: 'pg',
pluginDivisionMode: 'schema',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
plugin: {
plugin2: {
client: 'mysql',
},
},
};
const testManager = DatabaseManager.fromConfig(
new ConfigReader(overrideConfig),
);
const pluginId = 'schemaoverride';
await testManager.forPlugin(pluginId).getClient();
}),
{
pg: connector1,
mysql: connector2,
},
);
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
await impl.forPlugin('plugin1').getClient();
expect(connector1.getClient).toHaveBeenCalledTimes(1);
expect(connector1.getClient).toHaveBeenLastCalledWith('plugin1', undefined);
expect(connector2.getClient).toHaveBeenCalledTimes(0);
expect(baseConfig.get()).toMatchObject({
client: 'pg',
connection: config.backend.database.connection,
});
await impl.forPlugin('plugin2').getClient();
expect(connector1.getClient).toHaveBeenCalledTimes(1);
expect(connector1.getClient).toHaveBeenLastCalledWith('plugin1', undefined);
expect(connector2.getClient).toHaveBeenCalledTimes(1);
expect(connector2.getClient).toHaveBeenLastCalledWith('plugin2', undefined);
});
expect(overrides).toMatchObject({
searchPath: [pluginId],
});
it('retains the migration skip info', async () => {
const connector = {
getClient: jest.fn(),
dropDatabase: jest.fn(),
} satisfies Connector;
const impl1 = new DatabaseManagerImpl(new ConfigReader({ client: 'pg' }), {
pg: connector,
});
it('plugin does not provide schema override for non pg client', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
pluginDivisionMode: 'schema',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
const pluginId = 'any-plugin';
await testManager.forPlugin(pluginId).getClient();
const impl2 = new DatabaseManagerImpl(
new ConfigReader({ client: 'pg' }),
{ pg: connector },
{ migrations: { skip: true } },
);
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
client: 'better-sqlite3',
connection: config.backend.database.connection,
});
expect(overrides).not.toHaveProperty('searchPath');
expect((await impl1.forPlugin('plugin1')).migrations).toEqual({
skip: false,
});
it('plugin does not provide schema override if pluginDivisionMode is set to database', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
pluginDivisionMode: 'database',
connection: 'some-file-path',
},
},
}),
);
const pluginId = 'any-plugin';
await testManager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
expect(overrides).not.toHaveProperty('searchPath');
});
it('plugin does not provide schema override if pluginDivisionMode is not set', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
const pluginId = 'schemaoverride';
await testManager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
expect(overrides).not.toHaveProperty('searchPath');
});
it('pluginDivisionMode ensures that each plugin schema exists', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
pluginDivisionMode: 'schema',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
const pluginId = 'testdbname';
await testManager.forPlugin(pluginId).getClient();
const mockCalls = mocked(ensureSchemaExists).mock.calls;
const [_, schemaName] = mockCalls[0];
expect(schemaName).toEqual('testdbname');
});
it('pluginDivisionMode allows connection overrides for plugins', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
pluginDivisionMode: 'schema',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
plugin: {
testdbname: {
connection: {
database: 'database_name_overridden',
host: 'newhost',
},
},
},
},
},
}),
);
const pluginId = 'testdbname';
await testManager.forPlugin(pluginId).getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
client: 'pg',
connection: {
database: 'database_name_overridden',
host: 'newhost',
user: 'foo',
password: 'bar',
},
});
expect(overrides).toHaveProperty('searchPath', ['testdbname']);
expect(overrides).toHaveProperty(
'connection.database',
'database_name_overridden',
);
});
it('ensureExists does not create database or schema when false', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
pluginDivisionMode: 'schema',
ensureExists: false,
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
const pluginId = 'testdbname';
await testManager.forPlugin(pluginId).getClient();
expect(mocked(ensureDatabaseExists)).toHaveBeenCalledTimes(0);
expect(mocked(ensureSchemaExists)).toHaveBeenCalledTimes(0);
});
it('fetches and merges additional knex config', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
database: 'foodb',
},
knexConfig: {
something: false,
},
plugin: {
testdbname: {
knexConfig: {
debug: true,
},
},
},
},
},
}),
);
await testManager.forPlugin('testdbname').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig] = mockCalls[0];
expect(baseConfig.data).toEqual(
expect.objectContaining({
debug: true,
something: false,
}),
);
});
it('sets the owner config for plugin using default config', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
database: 'foodb',
},
role: 'backstage',
plugin: {
testowner: {},
},
},
},
}),
);
await testManager.forPlugin('testowner').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig] = mockCalls[0];
expect(baseConfig.data.role).toEqual('backstage');
});
it('sets the owner config for plugin using plugin config', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
database: 'foodb',
},
role: 'backstage',
plugin: {
testowner: {
role: 'backstage-plugin',
},
},
},
},
}),
);
await testManager.forPlugin('testowner').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig] = mockCalls[0];
expect(baseConfig.data.role).toEqual('backstage-plugin');
});
it('Defaults the application_name for postgres clients', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {},
},
},
}),
);
await testManager.forPlugin('testplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, _] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
connection: {
application_name: 'backstage_plugin_testplugin',
},
});
});
it('Allows manually setting the application_name for postgres clients', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
application_name: 'backstage_custom_app_name',
},
},
},
}),
);
await testManager.forPlugin('testplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, overrides] = mockCalls[0];
expect(baseConfig.get().connection.application_name).toBe(
'backstage_custom_app_name',
);
expect(overrides.connection.application_name).toBeUndefined();
});
it('Allows manually setting the application_name for individual plugin client', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
application_name: 'backstage_custom_app_name',
},
plugin: {
overrideplugin: {
connection: {
application_name: 'custom_plugin',
},
},
},
},
},
}),
);
await testManager.forPlugin('overrideplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [baseConfig, _] = mockCalls[0];
expect(baseConfig.get().connection.application_name).toBe(
'custom_plugin',
);
expect((await impl2.forPlugin('plugin1')).migrations).toEqual({
skip: true,
});
});
});
packages/backend-common/src/database/DatabaseManager.ts
+105 -273
View File
@@ -14,28 +14,19 @@
* limitations under the License.
*/
import { Config, ConfigReader } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { Knex } from 'knex';
import { merge, omit } from 'lodash';
import { mergeDatabaseConfig } from './config';
import {
createDatabaseClient,
createNameOverride,
createSchemaOverride,
ensureDatabaseExists,
ensureSchemaExists,
normalizeConnection,
} from './connection';
import { PluginDatabaseManager } from './types';
import path from 'path';
import {
DatabaseService,
LifecycleService,
LoggerService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config } from '@backstage/config';
import { stringifyError } from '@backstage/errors';
import { Knex } from 'knex';
import { MysqlConnector } from './connectors/mysql';
import { PgConnector } from './connectors/postgres';
import { Sqlite3Connector } from './connectors/sqlite3';
import { Connector, PluginDatabaseManager } from './types';
/**
* Provides a config lookup path for a plugin's config block.
@@ -63,40 +54,12 @@ export type LegacyRootDatabaseService = {
};
/**
* Manages database connections for Backstage backend plugins.
*
* @public
* @remarks
*
* The database manager allows the user to set connection and client settings on
* a per pluginId basis by defining a database config block under
* `plugin.<pluginId>` in addition to top level defaults. Optionally, a user may
* set `prefix` which is used to prefix generated database names if config is
* not provided.
* Testable implementation class for {@link DatabaseManager} below.
*/
export class DatabaseManager implements LegacyRootDatabaseService {
/**
* Creates a {@link DatabaseManager} from `backend.database` config.
*
* @param config - The loaded application configuration.
* @param options - An optional configuration object.
*/
static fromConfig(
config: Config,
options?: DatabaseManagerOptions,
): DatabaseManager {
const databaseConfig = config.getConfig('backend.database');
return new DatabaseManager(
databaseConfig,
databaseConfig.getOptionalString('prefix'),
options,
);
}
private constructor(
export class DatabaseManagerImpl implements LegacyRootDatabaseService {
constructor(
private readonly config: Config,
private readonly prefix: string = 'backstage_plugin_',
private readonly connectors: Record<string, Connector>,
private readonly options?: DatabaseManagerOptions,
private readonly databaseCache: Map<string, Promise<Knex>> = new Map(),
) {}
@@ -115,52 +78,18 @@ export class DatabaseManager implements LegacyRootDatabaseService {
pluginMetadata: PluginMetadataService;
},
): PluginDatabaseManager {
const getClient = () => this.getDatabase(pluginId, deps);
const client = this.getClientType(pluginId).client;
const connector = this.connectors[client];
if (!connector) {
throw new Error(
`Unsupported database client type '${client}' specified for plugin '${pluginId}'`,
);
}
const getClient = () => this.getDatabase(pluginId, connector, deps);
const migrations = { skip: false, ...this.options?.migrations };
return { getClient, migrations };
}
/**
* Provides the canonical database name for a given plugin.
*
* This method provides the effective database name which is determined using global
* and plugin specific database config. If no explicit database name is configured
* and `pluginDivisionMode` is not `schema`, this method will provide a generated name
* which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is
* `schema`, it will fallback to using the default database for the knex instance.
*
* @param pluginId - Lookup the database name for given plugin
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
if (this.getClientType(pluginId).client.includes('sqlite3')) {
const sqliteFilename: string | undefined = (
connection as Knex.Sqlite3ConnectionConfig
).filename;
if (sqliteFilename === ':memory:') {
return sqliteFilename;
}
const sqliteDirectory =
(connection as { directory?: string }).directory ?? '.';
return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`);
}
const databaseName = (connection as Knex.ConnectionConfig)?.database;
// `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database
if (this.getPluginDivisionModeConfig() === 'schema') {
return databaseName;
}
// all other supported databases should fallback to an auto-prefixed name
return databaseName ?? `${this.prefix}${pluginId}`;
}
/**
* Provides the client type which should be used for a given plugin.
*
@@ -188,143 +117,6 @@ export class DatabaseManager implements LegacyRootDatabaseService {
};
}
private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}
/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();
const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();
return merge(baseConfig, pluginConfig);
}
private getEnsureExistsConfig(pluginId: string): boolean {
const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true;
return (
this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ??
baseConfig
);
}
private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}
/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted for
* all supported databases excluding SQLite unless `pluginDivisionMode` is set
* to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);
let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);
if (
client.includes('sqlite3') &&
'filename' in baseConnection &&
baseConnection.filename !== ':memory:'
) {
throw new Error(
'`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.',
);
}
// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`. SQLite3's `filename` property is an exception as this is used as a
// directory elsewhere so we preserve `filename`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}
// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);
if (client === 'pg') {
(
baseConnection as Knex.PgConnectionConfig
).application_name ||= `backstage_plugin_${pluginId}`;
}
return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}
/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);
return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}
/**
* Provides a partial `Knex.Config` database schema override for a given
* plugin.
*
* @param pluginId - Target plugin to get database schema override
* @returns Partial `Knex.Config` with database schema override
*/
private getSchemaOverrides(pluginId: string): Knex.Config | undefined {
return createSchemaOverride(this.getClientType(pluginId).client, pluginId);
}
/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}
/**
* Provides a scoped Knex client for a plugin as per application config.
*
@@ -334,6 +126,7 @@ export class DatabaseManager implements LegacyRootDatabaseService {
*/
private async getDatabase(
pluginId: string,
connector: Connector,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
@@ -343,55 +136,13 @@ export class DatabaseManager implements LegacyRootDatabaseService {
return this.databaseCache.get(pluginId)!;
}
const clientPromise = Promise.resolve().then(async () => {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
);
const databaseName = this.getDatabaseName(pluginId);
if (databaseName && this.getEnsureExistsConfig(pluginId)) {
try {
await ensureDatabaseExists(pluginConfig, databaseName);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`,
);
}
}
let schemaOverrides;
if (this.getPluginDivisionModeConfig() === 'schema') {
schemaOverrides = this.getSchemaOverrides(pluginId);
if (this.getEnsureExistsConfig(pluginId)) {
try {
await ensureSchemaExists(pluginConfig, pluginId);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`,
);
}
}
}
const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
schemaOverrides,
);
const client = createDatabaseClient(
pluginConfig,
databaseClientOverrides,
deps,
);
if (process.env.NODE_ENV !== 'test') {
this.startKeepaliveLoop(pluginId, client);
}
return client;
});
const clientPromise = connector.getClient(pluginId, deps);
this.databaseCache.set(pluginId, clientPromise);
if (process.env.NODE_ENV !== 'test') {
clientPromise.then(client => this.startKeepaliveLoop(pluginId, client));
}
return clientPromise;
}
@@ -419,3 +170,84 @@ export class DatabaseManager implements LegacyRootDatabaseService {
}, 60 * 1000);
}
}
// NOTE: This class looks odd but is kept around for API compatibility reasons
/**
* Manages database connections for Backstage backend plugins.
*
* @public
* @remarks
*
* The database manager allows the user to set connection and client settings on
* a per pluginId basis by defining a database config block under
* `plugin.<pluginId>` in addition to top level defaults. Optionally, a user may
* set `prefix` which is used to prefix generated database names if config is
* not provided.
*/
export class DatabaseManager implements LegacyRootDatabaseService {
/**
* Creates a {@link DatabaseManager} from `backend.database` config.
*
* @param config - The loaded application configuration.
* @param options - An optional configuration object.
*/
static fromConfig(
config: Config,
options?: DatabaseManagerOptions,
): DatabaseManager {
const databaseConfig = config.getConfig('backend.database');
const prefix =
databaseConfig.getOptionalString('prefix') || 'backstage_plugin_';
return new DatabaseManager(
new DatabaseManagerImpl(
databaseConfig,
{
pg: new PgConnector(databaseConfig, prefix),
sqlite3: new Sqlite3Connector(databaseConfig),
'better-sqlite3': new Sqlite3Connector(databaseConfig),
mysql: new MysqlConnector(databaseConfig, prefix),
mysql2: new MysqlConnector(databaseConfig, prefix),
},
options,
),
);
}
private constructor(private readonly impl: DatabaseManagerImpl) {}
/**
* Generates a PluginDatabaseManager for consumption by plugins.
*
* @param pluginId - The plugin that the database manager should be created for. Plugin names
* should be unique as they are used to look up database config overrides under
* `backend.database.plugin`.
*/
forPlugin(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): PluginDatabaseManager {
return this.impl.forPlugin(pluginId, deps);
}
}
/**
* Helper for deleting databases, only exists for backend-test-utils for now.
*
* @public
*/
export async function dropDatabase(
dbConfig: Config,
...databaseNames: string[]
): Promise<void> {
const client = dbConfig.getString('client');
const prefix = dbConfig.getOptionalString('prefix') || 'backstage_plugin_';
if (client === 'pg') {
await new PgConnector(dbConfig, prefix).dropDatabase(...databaseNames);
} else if (client === 'mysql' || client === 'mysql2') {
await new MysqlConnector(dbConfig, prefix).dropDatabase(...databaseNames);
}
}
packages/backend-common/src/database/connection.test.ts
-308
View File
@@ -1,308 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
createDatabaseClient,
createNameOverride,
createSchemaOverride,
dropDatabase,
ensureSchemaExists,
parseConnectionString,
} from './connection';
import { mysqlConnector, pgConnector } from './connectors';
const mocked = (f: Function) => f as jest.Mock;
jest.mock('./connectors', () => {
const connectors = jest.requireActual('./connectors');
return {
...connectors,
mysqlConnector: {
...connectors.mysqlConnector,
dropDatabase: jest.fn(),
},
pgConnector: {
...connectors.pgConnector,
dropDatabase: jest.fn(),
ensureSchemaExists: jest.fn(),
},
};
});
describe('database connection', () => {
describe('createDatabaseClient', () => {
it('returns a postgres connection', () => {
expect(
createDatabaseClient(
new ConfigReader({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
database: 'foodb',
},
}),
),
).toBeTruthy();
});
it('returns an sqlite connection', () => {
expect(
createDatabaseClient(
new ConfigReader({
client: 'better-sqlite3',
connection: ':memory:',
}),
),
).toBeTruthy();
});
it('returns a mysql connection', () => {
expect(() =>
createDatabaseClient(
new ConfigReader({
client: 'mysql2',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
}),
),
).toBeTruthy();
});
it('accepts overrides', () => {
expect(
createDatabaseClient(
new ConfigReader({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
database: 'foodb',
},
}),
{
connection: {
database: 'foo',
},
},
),
).toBeTruthy();
});
it('throws an error without a client', () => {
expect(() =>
createDatabaseClient(
new ConfigReader({
connection: '',
}),
),
).toThrow();
});
it('throws an error without a connection', () => {
expect(() =>
createDatabaseClient(
new ConfigReader({
client: 'pg',
}),
),
).toThrow();
});
});
describe('createNameOverride', () => {
it('returns Knex config for postgres', () => {
expect(createNameOverride('pg', 'testpg')).toHaveProperty(
'connection.database',
'testpg',
);
});
it('returns Knex config for sqlite', () => {
expect(createNameOverride('better-sqlite3', 'testsqlite')).toHaveProperty(
'connection.filename',
'testsqlite',
);
});
it('returns Knex config for mysql', () => {
expect(createNameOverride('mysql', 'testmysql')).toHaveProperty(
'connection.database',
'testmysql',
);
});
it('throws an error for unknown connection', () => {
expect(() => createNameOverride('unknown', 'testname')).toThrow();
});
});
describe('parseConnectionString', () => {
it('returns parsed Knex.StaticConnectionConfig for postgres', () => {
expect(
parseConnectionString('postgresql://foo:bar@acme:5432/foodb', 'pg'),
).toHaveProperty('database', 'foodb');
});
it('returns parsed Knex.StaticConnectionConfig for mysql2', () => {
expect(
parseConnectionString('mysql://foo:bar@acme:3306/foodb', 'mysql2'),
).toHaveProperty('database', 'foodb');
});
it('throws an error if client hint is not provided', () => {
expect(() => parseConnectionString('sqlite://')).toThrow();
});
});
describe('createSchemaOverride', () => {
it('returns Knex config for postgres', () => {
expect(createSchemaOverride('pg', 'testpg')).toHaveProperty(
'searchPath',
['testpg'],
);
});
it('throws error for sqlite', () => {
expect(
createSchemaOverride('better-sqlite3', 'testsqlite'),
).toBeUndefined();
});
it('returns Knex config for mysql', () => {
expect(createSchemaOverride('mysql', 'testmysql')).toBeUndefined();
});
it('throws an error for unknown connection', () => {
expect(createSchemaOverride('unknown', 'testname')).toBeUndefined();
});
});
describe('ensureSchemaExists', () => {
it('returns successfully with pg client', async () => {
await ensureSchemaExists(
new ConfigReader({
client: 'pg',
schema: 'catalog',
connection: 'postgresql://testuser:testpass@acme:5432/userdbname',
}),
'catalog',
);
const mockCalls = mocked(
pgConnector.ensureSchemaExists as Function,
).mock.calls.splice(-1);
const [baseConfig, schemaName] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
client: 'pg',
connection: 'postgresql://testuser:testpass@acme:5432/userdbname',
});
expect(schemaName).toEqual('catalog');
});
it('throws error for non pg client', () => {
return expect(
ensureSchemaExists(
new ConfigReader({
client: 'better-sqlite3',
schema: 'catalog',
connection: ':memory:',
}),
'catalog',
),
).resolves.toBeUndefined();
});
});
describe('dropDatabase', () => {
it('returns successfully with pg client', async () => {
await dropDatabase(
new ConfigReader({
client: 'pg',
schema: 'catalog',
connection: 'postgresql://testuser:testpass@acme:5432/userdbname',
}),
'backstage_plugin_foobar',
);
const mockCalls = mocked(
pgConnector.dropDatabase as Function,
).mock.calls.splice(-1);
const [baseConfig, databaseName] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
client: 'pg',
connection: 'postgresql://testuser:testpass@acme:5432/userdbname',
});
expect(databaseName).toEqual('backstage_plugin_foobar');
});
it('returns successfully with mysql client', async () => {
await dropDatabase(
new ConfigReader({
client: 'mysql2',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
}),
'backstage_plugin_foobar',
);
const mockCalls = mocked(
mysqlConnector.dropDatabase as Function,
).mock.calls.splice(-1);
const [baseConfig, databaseName] = mockCalls[0];
expect(baseConfig.get()).toMatchObject({
client: 'mysql2',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
});
expect(databaseName).toEqual('backstage_plugin_foobar');
});
it('does nothing in other database drivers', () => {
return expect(
dropDatabase(
new ConfigReader({
client: 'better-sqlite3',
schema: 'catalog',
connection: ':memory:',
}),
'catalog',
),
).resolves.toBeUndefined();
});
});
});
packages/backend-common/src/database/connection.ts
-203
View File
@@ -1,203 +0,0 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { JsonObject } from '@backstage/types';
import { InputError } from '@backstage/errors';
import knexFactory, { Knex } from 'knex';
import limiterFactory from 'p-limit';
import { mergeDatabaseConfig } from './config';
import { DatabaseConnector } from './types';
import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors';
import {
LifecycleService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
type DatabaseClient =
| 'pg'
| 'better-sqlite3'
| 'sqlite3'
| 'mysql'
| 'mysql2'
| string;
// This limits the number of concurrent CREATE DATABASE and CREATE SCHEMA
// commands, globally, to just one. This is overly defensive, and was added as
// an attempt to counteract the pool issues on recent node versions. See
// https://github.com/backstage/backstage/pull/19988
const ddlLimiter = limiterFactory(1);
/**
* Mapping of client type to supported database connectors
*
* Database connectors can be aliased here, for example mysql2 uses
* the same connector as mysql.
*/
const ConnectorMapping: Record<DatabaseClient, DatabaseConnector> = {
pg: pgConnector,
'better-sqlite3': sqlite3Connector,
sqlite3: sqlite3Connector,
mysql: mysqlConnector,
mysql2: mysqlConnector,
};
/**
* Creates a knex database connection
*
* @public
* @param dbConfig - The database config
* @param overrides - Additional options to merge with the config
*/
export function createDatabaseClient(
dbConfig: Config,
overrides?: Partial<Knex.Config>,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
) {
const client: DatabaseClient = dbConfig.getString('client');
return (
ConnectorMapping[client]?.createClient(dbConfig, overrides, deps) ??
knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides))
);
}
/**
* Ensures that the given databases all exist, creating them if they do not.
*
* @public
*/
export async function ensureDatabaseExists(
dbConfig: Config,
...databases: Array<string>
): Promise<void> {
const client: DatabaseClient = dbConfig.getString('client');
return await ddlLimiter(() =>
ConnectorMapping[client]?.ensureDatabaseExists?.(dbConfig, ...databases),
);
}
/**
* Drops the given databases.
*
* @public
*/
export async function dropDatabase(
dbConfig: Config,
...databases: Array<string>
): Promise<void> {
const client: DatabaseClient = dbConfig.getString('client');
return await ddlLimiter(() =>
ConnectorMapping[client]?.dropDatabase?.(dbConfig, ...databases),
);
}
/**
* Ensures that the given schemas all exist, creating them if they do not.
*
* @public
*/
export async function ensureSchemaExists(
dbConfig: Config,
...schemas: Array<string>
): Promise<void> {
const client: DatabaseClient = dbConfig.getString('client');
return await ddlLimiter(() =>
ConnectorMapping[client]?.ensureSchemaExists?.(dbConfig, ...schemas),
);
}
/**
* Provides a `Knex.Config` object with the provided database name for a given
* client.
*/
export function createNameOverride(
client: string,
name: string,
): Partial<Knex.Config> {
try {
return ConnectorMapping[client].createNameOverride(name);
} catch (e) {
throw new InputError(
`Unable to create database name override for '${client}' connector`,
e,
);
}
}
/**
* Provides a `Knex.Config` object with the provided database schema for a given
* client. Currently only supported by `pg`.
*/
export function createSchemaOverride(
client: string,
name: string,
): Partial<Knex.Config | undefined> {
try {
return ConnectorMapping[client]?.createSchemaOverride?.(name);
} catch (e) {
throw new InputError(
`Unable to create database schema override for '${client}' connector`,
e,
);
}
}
/**
* Parses a connection string for a given client and provides a connection config.
*/
export function parseConnectionString(
connectionString: string,
client?: string,
): Knex.StaticConnectionConfig {
if (typeof client === 'undefined' || client === null) {
throw new InputError(
'Database connection string client type auto-detection is not yet supported.',
);
}
try {
return ConnectorMapping[client].parseConnectionString(connectionString);
} catch (e) {
throw new InputError(
`Unable to parse connection string for '${client}' connector`,
);
}
}
/**
* Normalizes a connection config or string into an object which can be passed
* to Knex.
*/
export function normalizeConnection(
connection: Knex.StaticConnectionConfig | JsonObject | string | undefined,
client: string,
): Partial<Knex.StaticConnectionConfig> {
if (typeof connection === 'undefined' || connection === null) {
return {};
}
return typeof connection === 'string' || connection instanceof String
? parseConnectionString(connection as string, client)
: connection;
}
packages/backend-common/src/database/connectors/defaultNameOverride.test.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import defaultNameOverride from './defaultNameOverride';
describe('defaultNameOverride()', () => {
packages/backend-common/src/database/connectors/defaultNameOverride.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex } from 'knex';
/**
packages/backend-common/src/database/connectors/defaultSchemaOverride.test.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import defaultSchemaOverride from './defaultSchemaOverride';
describe('defaultNameOverride()', () => {
packages/backend-common/src/database/connectors/defaultSchemaOverride.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex } from 'knex';
/**
packages/backend-common/src/database/connectors/index.ts
+1
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './mysql';
export * from './postgres';
export * from './sqlite3';
packages/backend-common/src/database/config.test.ts → packages/backend-common/src/database/connectors/mergeDatabaseConfig.test.ts
+2 -2
View File
@@ -14,9 +14,9 @@
* limitations under the License.
*/
import { mergeDatabaseConfig } from './config';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
describe('config', () => {
describe('mergeDatabaseConfig', () => {
describe('mergeDatabaseConfig', () => {
it('does not mutate the input object', () => {
const input = {
packages/backend-common/src/database/config.ts → packages/backend-common/src/database/connectors/mergeDatabaseConfig.ts
View File
packages/backend-common/src/database/connectors/mysql.ts
+257 -8
View File
@@ -14,14 +14,23 @@
* limitations under the License.
*/
import knexFactory, { Knex } from 'knex';
import yn from 'yn';
import { Config } from '@backstage/config';
import {
LifecycleService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config, ConfigReader } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import { JsonObject } from '@backstage/types';
import knexFactory, { Knex } from 'knex';
import { merge, omit } from 'lodash';
import limiterFactory from 'p-limit';
import yn from 'yn';
import { Connector, DatabaseConnector } from '../types';
import defaultNameOverride from './defaultNameOverride';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
// Limits the number of concurrent DDL operations to 1
const ddlLimiter = limiterFactory(1);
/**
* Creates a knex mysql database connection
@@ -169,7 +178,7 @@ export async function ensureMysqlDatabaseExists(
let lastErr: Error | undefined = undefined;
for (let i = 0; i < 3; i++) {
try {
return await ensureDatabase(database);
return await ddlLimiter(() => ensureDatabase(database));
} catch (err) {
lastErr = err;
}
@@ -209,7 +218,7 @@ export async function dropMysqlDatabase(
};
await Promise.all(
databases.map(async database => {
return await dropDatabase(database);
return await ddlLimiter(() => dropDatabase(database));
}),
);
} finally {
@@ -229,3 +238,243 @@ export const mysqlConnector: DatabaseConnector = Object.freeze({
parseConnectionString: parseMysqlConnectionString,
dropDatabase: dropMysqlDatabase,
});
/**
* Provides a config lookup path for a plugin's config block.
*/
function pluginPath(pluginId: string): string {
return `plugin.${pluginId}`;
}
function normalizeConnection(
connection: Knex.StaticConnectionConfig | JsonObject | string | undefined,
client: string,
): Partial<Knex.StaticConnectionConfig> {
if (typeof connection === 'undefined' || connection === null) {
return {};
}
return typeof connection === 'string' || connection instanceof String
? mysqlConnector.parseConnectionString(connection as string, client)
: connection;
}
function createNameOverride(
client: string,
name: string,
): Partial<Knex.Config> {
try {
return mysqlConnector.createNameOverride(name);
} catch (e) {
throw new InputError(
`Unable to create database name override for '${client}' connector`,
e,
);
}
}
export class MysqlConnector implements Connector {
constructor(
private readonly config: Config,
private readonly prefix: string,
) {}
async getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex> {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
);
const databaseName = this.getDatabaseName(pluginId);
if (databaseName && this.getEnsureExistsConfig(pluginId)) {
try {
await mysqlConnector.ensureDatabaseExists!(pluginConfig, databaseName);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`,
);
}
}
const pluginDivisionMode = this.getPluginDivisionModeConfig();
if (pluginDivisionMode !== 'database') {
throw new Error(
`The MySQL driver does not suppoert plugin division mode '${pluginDivisionMode}'`,
);
}
const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
);
const client = mysqlConnector.createClient(
pluginConfig,
databaseClientOverrides,
deps,
);
return client;
}
async dropDatabase(...databaseNames: string[]): Promise<void> {
return await dropMysqlDatabase(this.config, ...databaseNames);
}
/**
* Provides the canonical database name for a given plugin.
*
* This method provides the effective database name which is determined using
* global and plugin specific database config. If no explicit database name,
* this method will provide a generated name which is the pluginId prefixed
* with 'backstage_plugin_'.
*
* @param pluginId - Lookup the database name for given plugin
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
const databaseName = (connection as Knex.ConnectionConfig)?.database;
return databaseName ?? `${this.prefix}${pluginId}`;
}
/**
* Provides the client type which should be used for a given plugin.
*
* The client type is determined by plugin specific config if present.
* Otherwise the base client is used as the fallback.
*
* @param pluginId - Plugin to get the client type for
* @returns Object with client type returned as `client` and boolean
* representing whether or not the client was overridden as
* `overridden`
*/
private getClientType(pluginId: string): {
client: string;
overridden: boolean;
} {
const pluginClient = this.config.getOptionalString(
`${pluginPath(pluginId)}.client`,
);
const baseClient = this.config.getString('client');
const client = pluginClient ?? baseClient;
return {
client,
overridden: client !== baseClient,
};
}
private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}
/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();
const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();
return merge(baseConfig, pluginConfig);
}
private getEnsureExistsConfig(pluginId: string): boolean {
const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true;
return (
this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ??
baseConfig
);
}
private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}
/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted
* unless `pluginDivisionMode` is set to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);
let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);
// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}
// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);
return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}
/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);
return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}
/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}
}
packages/backend-common/src/database/connectors/postgres.test.ts
+1 -1
View File
@@ -16,8 +16,8 @@
import { Config, ConfigReader } from '@backstage/config';
import {
createPgDatabaseClient,
buildPgDatabaseConfig,
createPgDatabaseClient,
getPgConnectionConfig,
parsePgConnectionString,
} from './postgres';
packages/backend-common/src/database/connectors/postgres.ts
+314 -18
View File
@@ -14,15 +14,24 @@
* limitations under the License.
*/
import {
LifecycleService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config, ConfigReader } from '@backstage/config';
import { ForwardedError, InputError } from '@backstage/errors';
import { JsonObject } from '@backstage/types';
import knexFactory, { Knex } from 'knex';
import { Config } from '@backstage/config';
import { ForwardedError } from '@backstage/errors';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import { merge, omit } from 'lodash';
import limiterFactory from 'p-limit';
import { Client } from 'pg';
import { Connector, DatabaseConnector } from '../types';
import defaultNameOverride from './defaultNameOverride';
import defaultSchemaOverride from './defaultSchemaOverride';
import { Client } from 'pg';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
// Limits the number of concurrent DDL operations to 1
const ddlLimiter = limiterFactory(1);
/**
* Creates a knex postgres database connection
@@ -154,7 +163,7 @@ export async function ensurePgDatabaseExists(
let lastErr: Error | undefined = undefined;
for (let i = 0; i < 3; i++) {
try {
return await ensureDatabase(database);
return await ddlLimiter(() => ensureDatabase(database));
} catch (err) {
lastErr = err;
}
@@ -193,7 +202,9 @@ export async function ensurePgSchemaExists(
}
};
await Promise.all(schemas.map(ensureSchema));
await Promise.all(
schemas.map(database => ddlLimiter(() => ensureSchema(database))),
);
} finally {
await admin.destroy();
}
@@ -210,18 +221,17 @@ export async function dropPgDatabase(
...databases: Array<string>
) {
const admin = createPgDatabaseClient(dbConfig);
await Promise.all(
databases.map(async database => {
await admin.raw(`DROP DATABASE ??`, [database]);
}),
);
try {
await Promise.all(
databases.map(async database => {
await ddlLimiter(() => admin.raw(`DROP DATABASE ??`, [database]));
}),
);
} finally {
await admin.destroy();
}
}
/**
* PostgreSQL database connector.
*
* Exposes database connector functionality via an immutable object.
*/
export const pgConnector: DatabaseConnector = Object.freeze({
createClient: createPgDatabaseClient,
ensureDatabaseExists: ensurePgDatabaseExists,
@@ -231,3 +241,289 @@ export const pgConnector: DatabaseConnector = Object.freeze({
parseConnectionString: parsePgConnectionString,
dropDatabase: dropPgDatabase,
});
/**
* Provides a config lookup path for a plugin's config block.
*/
function pluginPath(pluginId: string): string {
return `plugin.${pluginId}`;
}
function normalizeConnection(
connection: Knex.StaticConnectionConfig | JsonObject | string | undefined,
client: string,
): Partial<Knex.StaticConnectionConfig> {
if (typeof connection === 'undefined' || connection === null) {
return {};
}
return typeof connection === 'string' || connection instanceof String
? pgConnector.parseConnectionString(connection as string, client)
: connection;
}
function createSchemaOverride(
client: string,
name: string,
): Partial<Knex.Config | undefined> {
try {
return pgConnector.createSchemaOverride?.(name);
} catch (e) {
throw new InputError(
`Unable to create database schema override for '${client}' connector`,
e,
);
}
}
function createNameOverride(
client: string,
name: string,
): Partial<Knex.Config> {
try {
return pgConnector.createNameOverride(name);
} catch (e) {
throw new InputError(
`Unable to create database name override for '${client}' connector`,
e,
);
}
}
export class PgConnector implements Connector {
constructor(
private readonly config: Config,
private readonly prefix: string,
) {}
async getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex> {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
);
const databaseName = this.getDatabaseName(pluginId);
if (databaseName && this.getEnsureExistsConfig(pluginId)) {
try {
await pgConnector.ensureDatabaseExists!(pluginConfig, databaseName);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that '${databaseName}' exists, ${error}`,
);
}
}
let schemaOverrides;
if (this.getPluginDivisionModeConfig() === 'schema') {
schemaOverrides = this.getSchemaOverrides(pluginId);
if (this.getEnsureExistsConfig(pluginId)) {
try {
await pgConnector.ensureSchemaExists!(pluginConfig, pluginId);
} catch (error) {
throw new Error(
`Failed to connect to the database to make sure that schema for plugin '${pluginId}' exists, ${error}`,
);
}
}
}
const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
schemaOverrides,
);
const client = pgConnector.createClient(
pluginConfig,
databaseClientOverrides,
deps,
);
return client;
}
async dropDatabase(...databaseNames: string[]): Promise<void> {
return await dropPgDatabase(this.config, ...databaseNames);
}
/**
* Provides the canonical database name for a given plugin.
*
* This method provides the effective database name which is determined using global
* and plugin specific database config. If no explicit database name is configured
* and `pluginDivisionMode` is not `schema`, this method will provide a generated name
* which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is
* `schema`, it will fallback to using the default database for the knex instance.
*
* @param pluginId - Lookup the database name for given plugin
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
const databaseName = (connection as Knex.ConnectionConfig)?.database;
// `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database
if (this.getPluginDivisionModeConfig() === 'schema') {
return databaseName;
}
// all other supported databases should fallback to an auto-prefixed name
return databaseName ?? `${this.prefix}${pluginId}`;
}
/**
* Provides the client type which should be used for a given plugin.
*
* The client type is determined by plugin specific config if present.
* Otherwise the base client is used as the fallback.
*
* @param pluginId - Plugin to get the client type for
* @returns Object with client type returned as `client` and boolean
* representing whether or not the client was overridden as
* `overridden`
*/
private getClientType(pluginId: string): {
client: string;
overridden: boolean;
} {
const pluginClient = this.config.getOptionalString(
`${pluginPath(pluginId)}.client`,
);
const baseClient = this.config.getString('client');
const client = pluginClient ?? baseClient;
return {
client,
overridden: client !== baseClient,
};
}
private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}
/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();
const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();
return merge(baseConfig, pluginConfig);
}
private getEnsureExistsConfig(pluginId: string): boolean {
const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true;
return (
this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ??
baseConfig
);
}
private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}
/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted
* unless `pluginDivisionMode` is set to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);
let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);
// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}
// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);
(
baseConnection as Knex.PgConnectionConfig
).application_name ||= `backstage_plugin_${pluginId}`;
return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}
/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);
return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}
/**
* Provides a partial `Knex.Config` database schema override for a given
* plugin.
*
* @param pluginId - Target plugin to get database schema override
* @returns Partial `Knex.Config` with database schema override
*/
private getSchemaOverrides(pluginId: string): Knex.Config | undefined {
return createSchemaOverride(this.getClientType(pluginId).client, pluginId);
}
/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}
}
packages/backend-common/src/database/connectors/sqlite3.ts
+252 -7
View File
@@ -14,17 +14,20 @@
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { ensureDirSync } from 'fs-extra';
import knexFactory, { Knex } from 'knex';
import path from 'path';
import { DevDataStore } from '@backstage/backend-dev-utils';
import { mergeDatabaseConfig } from '../config';
import { DatabaseConnector } from '../types';
import {
LifecycleService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config, ConfigReader } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { JsonObject } from '@backstage/types';
import { ensureDirSync } from 'fs-extra';
import knexFactory, { Knex } from 'knex';
import { merge, omit } from 'lodash';
import path from 'path';
import { Connector, DatabaseConnector } from '../types';
import { mergeDatabaseConfig } from './mergeDatabaseConfig';
/**
* Creates a knex SQLite3 database connection
@@ -159,8 +162,250 @@ export function parseSqliteConnectionString(
*
* Exposes database connector functionality via an immutable object.
*/
export const sqlite3Connector: DatabaseConnector = Object.freeze({
export const sqliteConnector: DatabaseConnector = Object.freeze({
createClient: createSqliteDatabaseClient,
createNameOverride: createSqliteNameOverride,
parseConnectionString: parseSqliteConnectionString,
});
/**
* Provides a config lookup path for a plugin's config block.
*/
function pluginPath(pluginId: string): string {
return `plugin.${pluginId}`;
}
function normalizeConnection(
connection: Knex.StaticConnectionConfig | JsonObject | string | undefined,
client: string,
): Partial<Knex.StaticConnectionConfig> {
if (typeof connection === 'undefined' || connection === null) {
return {};
}
return typeof connection === 'string' || connection instanceof String
? sqliteConnector.parseConnectionString(connection as string, client)
: connection;
}
function createNameOverride(
client: string,
name: string,
): Partial<Knex.Config> {
try {
return sqliteConnector.createNameOverride(name);
} catch (e) {
throw new InputError(
`Unable to create database name override for '${client}' connector`,
e,
);
}
}
export class Sqlite3Connector implements Connector {
constructor(private readonly config: Config) {}
async getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex> {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
);
const pluginDivisionMode = this.getPluginDivisionModeConfig();
if (pluginDivisionMode !== 'database') {
throw new Error(
`The SQLite driver does not suppoert plugin division mode '${pluginDivisionMode}'`,
);
}
const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
);
const client = sqliteConnector.createClient(
pluginConfig,
databaseClientOverrides,
deps,
);
return client;
}
async dropDatabase(..._databaseNames: string[]): Promise<void> {
// do nothing
}
/**
* Provides the canonical database name for a given plugin.
*
* This method provides the effective database name which is determined using global
* and plugin specific database config. If no explicit database name is configured
* and `pluginDivisionMode` is not `schema`, this method will provide a generated name
* which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is
* `schema`, it will fallback to using the default database for the knex instance.
*
* @param pluginId - Lookup the database name for given plugin
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
const sqliteFilename: string | undefined = (
connection as Knex.Sqlite3ConnectionConfig
).filename;
if (sqliteFilename === ':memory:') {
return sqliteFilename;
}
const sqliteDirectory =
(connection as { directory?: string }).directory ?? '.';
return path.join(sqliteDirectory, sqliteFilename ?? `${pluginId}.sqlite`);
}
/**
* Provides the client type which should be used for a given plugin.
*
* The client type is determined by plugin specific config if present.
* Otherwise the base client is used as the fallback.
*
* @param pluginId - Plugin to get the client type for
* @returns Object with client type returned as `client` and boolean
* representing whether or not the client was overridden as
* `overridden`
*/
private getClientType(pluginId: string): {
client: string;
overridden: boolean;
} {
const pluginClient = this.config.getOptionalString(
`${pluginPath(pluginId)}.client`,
);
const baseClient = this.config.getString('client');
const client = pluginClient ?? baseClient;
return {
client,
overridden: client !== baseClient,
};
}
private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}
/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();
const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();
return merge(baseConfig, pluginConfig);
}
private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}
/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted for
* all supported databases excluding SQLite unless `pluginDivisionMode` is set
* to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);
let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);
if (
client.includes('sqlite3') &&
'filename' in baseConnection &&
baseConnection.filename !== ':memory:'
) {
throw new Error(
'`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.',
);
}
// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`. SQLite3's `filename` property is an exception as this is used as a
// directory elsewhere so we preserve `filename`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}
// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);
return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}
/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);
return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}
/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}
}
packages/backend-common/src/database/index.ts
+5 -11
View File
@@ -14,17 +14,11 @@
* limitations under the License.
*/
export * from './DatabaseManager';
/*
* Undocumented API surface from connection is being reduced for future deprecation.
* Avoid exporting additional symbols.
*/
export {
createDatabaseClient,
ensureDatabaseExists,
dropDatabase,
} from './connection';
export { DatabaseManager, dropDatabase } from './DatabaseManager';
export type {
DatabaseManagerOptions,
LegacyRootDatabaseService,
} from './DatabaseManager';
export type { PluginDatabaseManager } from './types';
export { isDatabaseConflictError } from './util';
packages/backend-common/src/database/types.ts
+28 -12
View File
@@ -24,11 +24,11 @@ import { Knex } from 'knex';
export type { DatabaseService as PluginDatabaseManager } from '@backstage/backend-plugin-api';
/**
* DatabaseConnector manages an underlying Knex database driver.
* Manages an underlying Knex database driver.
*/
export interface DatabaseConnector {
/**
* createClient provides an instance of a knex database connector.
* Provides an instance of a knex database connector.
*/
createClient(
dbConfig: Config,
@@ -38,27 +38,29 @@ export interface DatabaseConnector {
pluginMetadata: PluginMetadataService;
},
): Knex;
/**
* createNameOverride provides a partial knex config sufficient to override a
* database name.
* Provides a partial knex config sufficient to override a database name.
*/
createNameOverride(name: string): Partial<Knex.Config>;
/**
* createSchemaOverride provides a partial knex config sufficient to override a
* PostgreSQL schema name within utilizing the `searchPath` knex configuration.
* Provides a partial knex config sufficient to override a PostgreSQL schema
* name within utilizing the `searchPath` knex configuration.
*/
createSchemaOverride?(name: string): Partial<Knex.Config>;
/**
* parseConnectionString produces a knex connection config object representing
* a database connection string.
* Produces a knex connection config object representing a database connection
* string.
*/
parseConnectionString(
connectionString: string,
client?: string,
): Knex.StaticConnectionConfig;
/**
* ensureDatabaseExists performs a side-effect to ensure database names passed in are
* present.
* Performs a side-effect to ensure database names passed in are present.
*
* Calling this function on databases which already exist should do nothing.
* Missing databases should be created if needed.
@@ -69,8 +71,7 @@ export interface DatabaseConnector {
): Promise<void>;
/**
* ensureSchemaExists performs a side-effect to ensure schema names passed in are
* present.
* Performs a side-effect to ensure schema names passed in are present.
*
* Calling this function on schemas which already exist should do nothing.
* Missing schemas should be created if needed.
@@ -80,5 +81,20 @@ export interface DatabaseConnector {
...schemas: Array<string>
): Promise<void>;
/**
* Deletes databases.
*/
dropDatabase?(dbConfig: Config, ...databases: Array<string>): Promise<void>;
}
export interface Connector {
getClient(
pluginId: string,
deps?: {
lifecycle: LifecycleService;
pluginMetadata: PluginMetadataService;
},
): Promise<Knex>;
dropDatabase(...databaseNames: string[]): Promise<void>;
}