From 5a3ce340728f4bc51aa58fc688db71f38f5e155e Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Tue, 25 May 2021 18:28:05 +0100 Subject: [PATCH] refactor: migrate from deprecated database manager Changes: - Swaps out `SingleConnectionDatabaseManager` to `DatabaseManager` across the repo. - Updates `backend-test-utils` to generate test plugin names prefixed with db to satisfy plugin naming constraint, e.g. 0 becomes db0. Signed-off-by: Minn Soe --- .changeset/five-donkeys-brake.md | 1 + .../tutorials/configuring-plugin-databases.md | 222 +++++++++++------- packages/backend-common/config.d.ts | 10 - .../src/database/TestDatabases.test.ts | 6 +- .../src/database/TestDatabases.ts | 12 +- .../backend-test-utils/src/database/types.ts | 5 +- .../default-app/packages/backend/src/index.ts | 4 +- .../src/service/CodeCoverageDatabase.test.ts | 4 +- .../src/service/router.test.ts | 4 +- .../tasks/StorageTaskBroker.test.ts | 7 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 9 +- .../src/service/router.test.ts | 4 +- 12 files changed, 168 insertions(+), 120 deletions(-) diff --git a/.changeset/five-donkeys-brake.md b/.changeset/five-donkeys-brake.md index d5bc695714..8a6523bb61 100644 --- a/.changeset/five-donkeys-brake.md +++ b/.changeset/five-donkeys-brake.md @@ -1,5 +1,6 @@ --- '@backstage/backend-common': minor +'@backstage/create-app': minor --- Deprecates `SingleConnectionDatabaseManager` and provides an API compatible database diff --git a/docs/tutorials/configuring-plugin-databases.md b/docs/tutorials/configuring-plugin-databases.md index b65addd7a8..27b800b3d2 100644 --- a/docs/tutorials/configuring-plugin-databases.md +++ b/docs/tutorials/configuring-plugin-databases.md @@ -1,42 +1,40 @@ --- id: configuring-plugin-databases -title: Configuring Plugin Specific Databases +title: Configuring Plugin Databases # prettier-ignore -description: Guide on how to use predefined databases for each plugin. +description: Guide on how to configure Backstage databases. --- -There are occasions where it may be difficult to deploy Backstage with -automatically created databases in production due to access control or other -restrictions. For example, your infrastructure might be defined as code using -tools such as Terraform or AWS CloudFormation where the name of each database is -defined, created and assigned explicitly. You may also need to use different -credentials for each database or use a set of credentials without the -permissions needed to create databases. +This guide covers a variety of production persistence use cases which are +supported out of the box by Backstage. The database manager allows the developer +to set the client and database connection details on a per plugin basis in +addition to the base client and connection configuration. This means that you +can use a SQLite 3 in-memory database for a specific plugin whilst using +PostgreSQL for everything else and so on. -`@backstage/backend-common` provides an alternate database manager, -`PluginConnectionDatabaseManager`, which allows the developer to set the client -and database connection on a per plugin basis in addition to the default client -and connection configuration. This means that you can use a `sqlite3` in memory -database for a specific plugin whilst using `postgres` for everything else and -so on. +By default, Backstage uses automatically created databases for each plugin whose +names follow the `backstage_plugin_` pattern, e.g. +`backstage_plugin_auth`. You can configure a different database name prefix for +use cases where you have multiple deployments running on a shared database +instance or cluster. -The database manager also allows you to change the database name prefix which is -used when a plugin database isn't explicitly configured. +With infrastructure defined as code or data (Terraform, AWS CloudFormation, +etc.), you may have database credentials which lack permissions to create new +databases or you do not have control over the database names. In these +instances, you can set the database name and connection information on a per +plugin basis as mentioned earlier. -There are two additional configuration options for this database manager: +Backstage supports all of these use cases with the `DatabaseManager` provided by +`@backstage/backend-common`. We will now cover how to use and configure +Backstage's databases. -- **`backend.database.prefix`:** is used to override the default - `backstage_plugin_` prefix which is used to generate a database name when it - is not explicitly set for that plugin. -- **`backend.database.plugin.`:** is used to define a `client` and - `connection` block for the plugin matching the `pluginId`, e.g. `catalog` is - the `pluginId` for the catalog plugin and any configuration defined under that - block is specific to that plugin. +## Prerequisites -## Install Database Drivers +### Dependencies -If you intend to use both `postgres` and `sqlite3`, you need to make sure the -appropriate database drivers are installed in your `backend` package. +Please ensure the appropriate database drivers are installed in your `backend` +package. If you intend to use both `postgres` and `sqlite3`, you can install +both of them. ```shell cd packages/backend @@ -51,48 +49,17 @@ yarn add sqlite3 From an operational perspective, you only need to install drivers for clients that are actively used. -## Add Configuration +### Database Manager -You can set the same type of values for `backend.database..client` and -`backend.database..connection` which are also accepted at the top -level. - -It is possible to override the default database name prefix, -`backstage_plugin_`, which is used when a name isn't explicitly defined. Set -`backend.database.prefix` as shown below. The database names for plugins such as -`catalog` and `auth` would now be `my_company_catalog` and `my_company_auth` -instead of `backstage_plugin_catalog` and `backstage_plugin_auth`. - -```yaml -backend: - database: - client: pg - prefix: my_company_ - connection: - host: localhost - user: postgres - password: password - plugin: - code-coverage: - connection: - database: pg_code_coverage_set_by_user -``` - -In the example above, the `code-coverage` plugin will use the same connection -configuration defined under `database.connection` and use -`pg_code_coverage_set_by_user` instead of `my_company_code-coverage` which would -be automatically generated if a plugin configuration wasn't explicitly set. - -## Integrate `PluginConnectionDatabaseManager` into `backend` - -The `SingleConnectionDatabaseManager` used by default should be replaced with -the `PluginConnectionDatabaseManager` in your `packages/backend/src/index.ts` -file. Import the manager and replace the `.fromConfig` call as shown below: +Existing Backstage instances should be updated to use `DatabaseManager` from +`@backstage/backend-common` in your `packages/backend/src/index.ts` file, the +`SingleConnectionDatabaseManager` has been deprecated. Import the manager and +update the references as shown below if this is not the case: ```diff import { - SingleConnectionDatabaseManager, -+ PluginConnectionDatabaseManager, ++ DatabaseManager, } from '@backstage/backend-common'; // ... @@ -100,24 +67,121 @@ import { function makeCreateEnv(config: Config) { // ... - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); -+ const databaseManager = PluginConnectionDatabaseManager.fromConfig(config); ++ const databaseManager = DatabaseManager.fromConfig(config); // ... } ``` +## Configuration + +You should set the base database client and connection information in your +`app-config.yaml` (or equivalent) file. The base client and configuration is +used as the default which is extended for each plugin with the same or unset +client type. If a client type is specified for a specific plugin which does not +match the base client, the configuration set for the plugin will be used as is +without extending the base configuration. + +Client type and configuration for plugins need to be defined under +**`backend.database.plugin.`**. As an example, `catalog` is the +`pluginId` for the catalog plugin and any configuration defined under that block +is specific to that plugin. We will now explore more detailed example +configurations below. + +### Minimal In-Memory Configuration + +In the example below, we are using `sqlite3` in-memory databases for all +plugins. You may want to use this configuration for testing or other non-durable +use cases. + +```yaml +backend: + database: + client: sqlite3 + connection: ':memory:' +``` + +### PostgreSQL + +The example below uses PostgreSQL (`pg`) as the database client for all plugins. +The `auth` plugin uses a user defined database name instead of the automatically +generated one which would have been `backstage_plugin_auth`. + +```yaml +backend: + database: + client: pg + connection: + host: some.example-pg-instance.tld + user: postgres + password: password + port: 5432 + plugin: + auth: + connection: + database: pg_auth_set_by_user +``` + +### Custom Database Name Prefix + +The configuration below uses `example_prefix_` as the database name prefix +instead of `backstage_plugin_`. Plugins such as `auth` and `catalog` will use +databases named `example_prefix_auth` and `example_prefix_catalog` respectively. + +```yaml +backend: + database: + client: pg + connection: + host: some.example-pg-instance.tld + user: postgres + password: password + port: 5432 + prefix: 'example_prefix_' +``` + +### Connection Configuration Per Plugin + +Both `auth` and `catalog` use connection configuration with different +credentials and database names. This type of configuration can be useful for +environments with infrastructure as code or data which may provide randomly +generated credentials and/or database names. + +```yaml +backend: + database: + client: pg + connection: 'postgresql://some.example-pg-instance.tld:5432' + plugin: + auth: + connection: 'postgresql://fort:knox@some.example-pg-instance.tld:5432/unwitting_fox_jumps' + catalog: + connection: 'postgresql://bank:reserve@some.example-pg-instance.tld:5432/shuffle_ransack_playback' +``` + +### PostgreSQL and SQLite 3 + +The example below uses PostgreSQL (`pg`) as the database client for all plugins +except the `auth` plugin which uses `sqlite3`. As the `auth` plugin's client +type is different from the base client type, the connection configuration for +`auth` is used verbatim without extending the base configuration for PostgreSQL. + +```yaml +backend: + database: + client: pg + connection: 'postgresql://foo:bar@some.example-pg-instance.tld:5432' + plugin: + auth: + client: sqlite3 + connection: ':memory:' +``` + ## Check Your Databases -The `PluginConnectionDatabaseManager` preserves the behaviour of the -`SingleConnectionDatabaseManager`. If the database does not exist, it will -attempt to create it. +The `DatabaseManager` will attempt to create the databases if they do not exist. +If you have set credentials per plugin because the credentials in the base +configuration do not have permissions to create databases, you must ensure they +exist before starting the service. The service will not be able to create them, +it can only use them. -If you are using this database manager to set the database name upfront because -the credentials do not have permissions to create databases, you must ensure -they exist before starting the service. The service will not be able to create -them, it can only use them. - -`sqlite3` databases do not need to be created upfront as with the existing -database manager. - -Your Backstage App can now use different database clients and configuration per -plugin! +Good luck! diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 3ada25dd8a..b42ce3eca5 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -14,16 +14,6 @@ * limitations under the License. */ -export type PluginDatabaseConfig = { - /** Database client to use. */ - client?: 'sqlite3' | 'pg'; - /** - * Database connection to use. - * @secret - */ - connection?: string | object; -}; - export interface Config { app: { baseUrl: string; // defined in core, but repeated here without doc diff --git a/packages/backend-test-utils/src/database/TestDatabases.test.ts b/packages/backend-test-utils/src/database/TestDatabases.test.ts index 7c1e6e1bdd..7a51111b02 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.test.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.test.ts @@ -71,7 +71,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -105,7 +105,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -139,7 +139,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_0'; + const database = 'backstage_plugin_db0'; const output = knexFactory({ client: 'mysql2', connection: { host, port, user, password, database }, diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts index ab5846d84d..0ba6bffc0b 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { Knex } from 'knex'; import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; @@ -142,7 +142,7 @@ export class TestDatabases { // Ensure that a unique logical database is created in the instance const connection = await instance.databaseManager - .forPlugin(String(this.lastDatabaseIndex++)) + .forPlugin(String(`db${this.lastDatabaseIndex++}`)) .getClient(); instance.connections.push(connection); @@ -157,7 +157,7 @@ export class TestDatabases { if (envVarName) { const connectionString = process.env[envVarName]; if (connectionString) { - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -195,7 +195,7 @@ export class TestDatabases { properties.dockerImageName!, ); - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -220,7 +220,7 @@ export class TestDatabases { properties.dockerImageName!, ); - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { @@ -241,7 +241,7 @@ export class TestDatabases { private async initSqlite( _properties: TestDatabaseProperties, ): Promise { - const databaseManager = SingleConnectionDatabaseManager.fromConfig( + const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts index 791cf09b7b..91b5939765 100644 --- a/packages/backend-test-utils/src/database/types.ts +++ b/packages/backend-test-utils/src/database/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { Knex } from 'knex'; /** @@ -35,10 +35,9 @@ export type TestDatabaseProperties = { export type Instance = { stopContainer?: () => Promise; - databaseManager: SingleConnectionDatabaseManager; + databaseManager: DatabaseManager; connections: Array; }; - export const allDatabases: Record< TestDatabaseId, TestDatabaseProperties diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index aebd034aae..70f4fb676e 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -14,7 +14,7 @@ import { useHotMemoize, notFoundHandler, CacheManager, - SingleConnectionDatabaseManager, + DatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; @@ -34,8 +34,8 @@ function makeCreateEnv(config: Config) { root.info(`Created UrlReader ${reader}`); - const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); const cacheManager = CacheManager.fromConfig(config); + const databaseManager = DatabaseManager.fromConfig(config); return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); diff --git a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts index 688eeb4ba2..fec2ea7afa 100644 --- a/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts +++ b/plugins/code-coverage-backend/src/service/CodeCoverageDatabase.test.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { DatabaseManager } from '@backstage/backend-common'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { @@ -22,7 +22,7 @@ import { } from './CodeCoverageDatabase'; import { JsonCodeCoverage } from './types'; -const db = SingleConnectionDatabaseManager.fromConfig( +const db = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/code-coverage-backend/src/service/router.test.ts b/plugins/code-coverage-backend/src/service/router.test.ts index e18cd75c2f..340502d1e6 100644 --- a/plugins/code-coverage-backend/src/service/router.test.ts +++ b/plugins/code-coverage-backend/src/service/router.test.ts @@ -20,14 +20,14 @@ import { getVoidLogger, PluginDatabaseManager, PluginEndpointDiscovery, - SingleConnectionDatabaseManager, + DatabaseManager, UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { createRouter } from './router'; function createDatabase(): PluginDatabaseManager { - return SingleConnectionDatabaseManager.fromConfig( + return DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 686dfc052f..5ed997bb25 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -14,17 +14,14 @@ * limitations under the License. */ -import { - getVoidLogger, - SingleConnectionDatabaseManager, -} from '@backstage/backend-common'; +import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; import { TaskSecrets, TaskSpec, DbTaskEventRow } from './types'; async function createStore(): Promise { - const manager = SingleConnectionDatabaseManager.fromConfig( + const manager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index c6ad1a8105..f8343e7d3c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -14,12 +14,9 @@ * limitations under the License. */ -import { - getVoidLogger, - SingleConnectionDatabaseManager, -} from '@backstage/backend-common'; -import { ConfigReader, JsonObject } from '@backstage/config'; import os from 'os'; +import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; +import { ConfigReader, JsonObject } from '@backstage/config'; import { createTemplateAction, TemplateActionRegistry } from '../actions'; import { RepoSpec } from '../actions/builtin/publish/util'; import { DatabaseTaskStore } from './DatabaseTaskStore'; @@ -27,7 +24,7 @@ import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; async function createStore(): Promise { - const manager = SingleConnectionDatabaseManager.fromConfig( + const manager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 9ca2eef9e6..9349e076fd 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -31,7 +31,7 @@ jest.doMock('fs-extra', () => ({ import { getVoidLogger, PluginDatabaseManager, - SingleConnectionDatabaseManager, + DatabaseManager, UrlReaders, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; @@ -47,7 +47,7 @@ const createCatalogClient = (templates: any[] = []) => } as CatalogApi); function createDatabase(): PluginDatabaseManager { - return SingleConnectionDatabaseManager.fromConfig( + return DatabaseManager.fromConfig( new ConfigReader({ backend: { database: {