From 5a8fcb4eb3f4c74806e70bd87a42c61a8dc3e4f1 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 23 Aug 2024 15:46:19 +0200 Subject: [PATCH 1/8] backend-defaults: Add config options to skip database migrations Signed-off-by: Johan Haals --- .changeset/wild-buses-notice.md | 5 + packages/backend-defaults/config.d.ts | 102 ++++++++++++++++++ .../database/DatabaseManager.test.ts | 30 +++++- .../entrypoints/database/DatabaseManager.ts | 12 ++- 4 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 .changeset/wild-buses-notice.md diff --git a/.changeset/wild-buses-notice.md b/.changeset/wild-buses-notice.md new file mode 100644 index 0000000000..7c67ec685b --- /dev/null +++ b/.changeset/wild-buses-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Added the option to skip database migrations by setting `skipMigrations: true` in options. This can be done globally in the database config or by plugin id. diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index fee8846c4b..35de7ccdac 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -375,6 +375,108 @@ export interface Config { }; }; + /** Database connection configuration, select base database type using the `client` field */ + database: { + /** Default database client to use */ + client: 'better-sqlite3' | 'sqlite3' | 'pg'; + /** + * Base database connection string, or object with individual connection properties + * @visibility secret + */ + connection: + | string + | { + /** + * Password that belongs to the client User + * @visibility secret + */ + password?: string; + /** + * Other connection settings + */ + [key: string]: unknown; + }; + /** Database name prefix override */ + prefix?: string; + /** + * Whether to ensure the given database exists by creating it if it does not. + * Defaults to true if unspecified. + */ + ensureExists?: boolean; + /** + * Whether to ensure the given database schema exists by creating it if it does not. + * Defaults to false if unspecified. + * + * NOTE: Currently only supported by the `pg` client when pluginDivisionMode: schema + */ + ensureSchemaExists?: boolean; + /** + * How plugins databases are managed/divided in the provided database instance. + * + * `database` -> Plugins are each given their own database to manage their schemas/tables. + * + * `schema` -> Plugins will be given their own schema (in the specified/default database) + * to manage their tables. + * + * NOTE: Currently only supported by the `pg` client. + * + * @default database + */ + pluginDivisionMode?: 'database' | 'schema'; + /** Configures the ownership of newly created schemas in pg databases. */ + role?: string; + /** + * Skip running database migrations. + * NOTE: Currently only supported by the `pg` client. + */ + skipMigrations?: boolean; + /** + * Arbitrary config object to pass to knex when initializing + * (https://knexjs.org/#Installation-client). Most notable is the debug + * and asyncStackTraces booleans + */ + knexConfig?: object; + /** Plugin specific database configuration and client override */ + plugin?: { + [pluginId: string]: { + /** Database client override */ + client?: 'better-sqlite3' | 'sqlite3' | 'pg'; + /** + * Database connection string or Knex object override + * @visibility secret + */ + connection?: string | object; + /** + * Whether to ensure the given database exists by creating it if it does not. + * Defaults to base config if unspecified. + */ + ensureExists?: boolean; + /** + * Whether to ensure the given database schema exists by creating it if it does not. + * Defaults to false if unspecified. + * + * NOTE: Currently only supported by the `pg` client when pluginDivisionMode: schema + */ + ensureSchemaExists?: boolean; + /** + * Arbitrary config object to pass to knex when initializing + * (https://knexjs.org/#Installation-client). Most notable is the + * debug and asyncStackTraces booleans. + * + * This is merged recursively into the base knexConfig + */ + knexConfig?: object; + /** Configures the ownership of newly created schemas in pg databases. */ + role?: string; + /** + * Skip running database migrations. + * NOTE: Currently only supported by the `pg` client. + */ + skipMigrations?: boolean; + }; + }; + }; + /** * Options used by the default discovery service. */ diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts index 347ec742c0..630dfa6061 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts @@ -107,9 +107,15 @@ describe('DatabaseManagerImpl', () => { }); const impl2 = new DatabaseManagerImpl( - new ConfigReader({ client: 'pg' }), - { pg: connector }, - { migrations: { skip: true } }, + new ConfigReader({ + client: 'pg', + backend: { + database: { plugin: { plugin1: { skipMigrations: true } } }, + }, + }), + { + pg: connector, + }, ); expect((await impl1.forPlugin('plugin1')).migrations).toEqual({ @@ -119,5 +125,23 @@ describe('DatabaseManagerImpl', () => { expect((await impl2.forPlugin('plugin1')).migrations).toEqual({ skip: true, }); + + const impl3 = new DatabaseManagerImpl( + new ConfigReader({ + client: 'pg', + backend: { + database: { + skipMigrations: true, + plugin: { plugin1: { skipMigrations: false } }, + }, + }, + }), + { + pg: connector, + }, + ); + expect((await impl3.forPlugin('plugin1')).migrations).toEqual({ + skip: true, + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts index 7e3adf17df..f756e11aac 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts @@ -41,7 +41,6 @@ function pluginPath(pluginId: string): string { * @public */ export type DatabaseManagerOptions = { - migrations?: DatabaseService['migrations']; logger?: LoggerService; }; @@ -86,7 +85,16 @@ export class DatabaseManagerImpl implements LegacyRootDatabaseService { ); } const getClient = () => this.getDatabase(pluginId, connector, deps); - const migrations = { skip: false, ...this.options?.migrations }; + + const migrationConfig = + this.config.getOptionalBoolean('backend.database.skipMigrations') || + this.config.getOptionalBoolean( + `backend.database.plugin.${pluginId}.skipMigrations`, + ); + const migrations = { + skip: migrationConfig || false, + }; + return { getClient, migrations }; } From 8e97689436a2bedc6ee5a8d8e5b269b0a050a06d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 10:11:21 +0200 Subject: [PATCH 2/8] chore: update api report Signed-off-by: Johan Haals --- packages/backend-defaults/api-report-database.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-defaults/api-report-database.md b/packages/backend-defaults/api-report-database.md index 10de8df1ff..d8cd2c7948 100644 --- a/packages/backend-defaults/api-report-database.md +++ b/packages/backend-defaults/api-report-database.md @@ -27,7 +27,6 @@ export class DatabaseManager implements LegacyRootDatabaseService { // @public export type DatabaseManagerOptions = { - migrations?: DatabaseService['migrations']; logger?: LoggerService; }; From 08f0f7fd22bd54e105984d12a121d92fc116027d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 11:16:37 +0200 Subject: [PATCH 3/8] Make sure options take precedence Signed-off-by: Johan Haals --- .../database/DatabaseManager.test.ts | 50 +++++++++++++++---- .../entrypoints/database/DatabaseManager.ts | 23 +++++---- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts index 630dfa6061..0d52ff16ff 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.test.ts @@ -96,17 +96,47 @@ describe('DatabaseManagerImpl', () => { expect(connector2.getClient).toHaveBeenLastCalledWith('plugin2', undefined); }); - it('retains the migration skip info', async () => { + it('migration skip options take precedence over config', async () => { const connector = { getClient: jest.fn(), dropDatabase: jest.fn(), } satisfies Connector; + const impl = new DatabaseManagerImpl( + new ConfigReader({ + client: 'pg', + backend: { + database: { + skipMigrations: true, + plugin: { plugin1: { skipMigrations: true } }, + }, + }, + }), + { + pg: connector, + }, + { migrations: { skip: false } }, + ); + expect((await impl.forPlugin('plugin1')).migrations).toEqual({ + skip: false, + }); + const impl1 = new DatabaseManagerImpl(new ConfigReader({ client: 'pg' }), { pg: connector, }); - const impl2 = new DatabaseManagerImpl( + expect((await impl1.forPlugin('plugin1')).migrations).toEqual({ + skip: false, + }); + }); + + it('plugin can skip migrations using config', async () => { + const connector = { + getClient: jest.fn(), + dropDatabase: jest.fn(), + } satisfies Connector; + + const impl = new DatabaseManagerImpl( new ConfigReader({ client: 'pg', backend: { @@ -118,15 +148,14 @@ describe('DatabaseManagerImpl', () => { }, ); - expect((await impl1.forPlugin('plugin1')).migrations).toEqual({ + expect((await impl.forPlugin('plugin1')).migrations).toEqual({ + skip: true, + }); + expect((await impl.forPlugin('plugin2')).migrations).toEqual({ skip: false, }); - expect((await impl2.forPlugin('plugin1')).migrations).toEqual({ - skip: true, - }); - - const impl3 = new DatabaseManagerImpl( + const impl2 = new DatabaseManagerImpl( new ConfigReader({ client: 'pg', backend: { @@ -140,7 +169,10 @@ describe('DatabaseManagerImpl', () => { pg: connector, }, ); - expect((await impl3.forPlugin('plugin1')).migrations).toEqual({ + expect((await impl2.forPlugin('plugin1')).migrations).toEqual({ + skip: false, + }); + expect((await impl2.forPlugin('plugin2')).migrations).toEqual({ skip: true, }); }); diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts index f756e11aac..5c39307f27 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts @@ -41,6 +41,7 @@ function pluginPath(pluginId: string): string { * @public */ export type DatabaseManagerOptions = { + migrations?: DatabaseService['migrations']; logger?: LoggerService; }; @@ -86,16 +87,20 @@ export class DatabaseManagerImpl implements LegacyRootDatabaseService { } const getClient = () => this.getDatabase(pluginId, connector, deps); - const migrationConfig = - this.config.getOptionalBoolean('backend.database.skipMigrations') || - this.config.getOptionalBoolean( - `backend.database.plugin.${pluginId}.skipMigrations`, - ); - const migrations = { - skip: migrationConfig || false, - }; + let skip = false; + // config options take precedence over config + if (this.options?.migrations?.skip !== undefined) { + skip = this.options.migrations.skip; + } else { + skip = + this.config.getOptionalBoolean( + `backend.database.plugin.${pluginId}.skipMigrations`, + ) ?? + this.config.getOptionalBoolean('backend.database.skipMigrations') ?? + false; + } - return { getClient, migrations }; + return { getClient, migrations: { skip } }; } /** From 6a6f45fde22ae0ad0eb54dda6050d37ef5b158c5 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 11:28:56 +0200 Subject: [PATCH 4/8] chore: update api report Signed-off-by: Johan Haals --- packages/backend-defaults/api-report-database.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-defaults/api-report-database.md b/packages/backend-defaults/api-report-database.md index d8cd2c7948..10de8df1ff 100644 --- a/packages/backend-defaults/api-report-database.md +++ b/packages/backend-defaults/api-report-database.md @@ -27,6 +27,7 @@ export class DatabaseManager implements LegacyRootDatabaseService { // @public export type DatabaseManagerOptions = { + migrations?: DatabaseService['migrations']; logger?: LoggerService; }; From 72e356e8b99d559ce7ea37472a171d8b5acbe9c8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 11:31:18 +0200 Subject: [PATCH 5/8] chore: tweak wording Signed-off-by: Johan Haals --- .../src/entrypoints/database/DatabaseManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts index 5c39307f27..42ba644823 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts @@ -88,7 +88,7 @@ export class DatabaseManagerImpl implements LegacyRootDatabaseService { const getClient = () => this.getDatabase(pluginId, connector, deps); let skip = false; - // config options take precedence over config + // class options take precedence over config if (this.options?.migrations?.skip !== undefined) { skip = this.options.migrations.skip; } else { From 1532b089b99693168bde74b8b6ec27055fda4e0d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 11:36:33 +0200 Subject: [PATCH 6/8] chore: simplify skip condition Signed-off-by: Johan Haals --- .../entrypoints/database/DatabaseManager.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts index 42ba644823..b3e69f0ae2 100644 --- a/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts +++ b/packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts @@ -87,18 +87,13 @@ export class DatabaseManagerImpl implements LegacyRootDatabaseService { } const getClient = () => this.getDatabase(pluginId, connector, deps); - let skip = false; - // class options take precedence over config - if (this.options?.migrations?.skip !== undefined) { - skip = this.options.migrations.skip; - } else { - skip = - this.config.getOptionalBoolean( - `backend.database.plugin.${pluginId}.skipMigrations`, - ) ?? - this.config.getOptionalBoolean('backend.database.skipMigrations') ?? - false; - } + const skip = + this.options?.migrations?.skip ?? + this.config.getOptionalBoolean( + `backend.database.plugin.${pluginId}.skipMigrations`, + ) ?? + this.config.getOptionalBoolean('backend.database.skipMigrations') ?? + false; return { getClient, migrations: { skip } }; } From 66b7af4217ae951fdbe87f666ccab5180cfc1e46 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 26 Aug 2024 13:40:14 +0200 Subject: [PATCH 7/8] chore: tweak wording, fix config docs Signed-off-by: Johan Haals --- .changeset/wild-buses-notice.md | 2 +- packages/backend-defaults/config.d.ts | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.changeset/wild-buses-notice.md b/.changeset/wild-buses-notice.md index 7c67ec685b..3620786aa8 100644 --- a/.changeset/wild-buses-notice.md +++ b/.changeset/wild-buses-notice.md @@ -2,4 +2,4 @@ '@backstage/backend-defaults': patch --- -Added the option to skip database migrations by setting `skipMigrations: true` in options. This can be done globally in the database config or by plugin id. +Added the option to skip database migrations by setting `skipMigrations: true` in config. This can be done globally in the database config or by plugin id. diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 35de7ccdac..a13e90e39f 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -425,10 +425,7 @@ export interface Config { pluginDivisionMode?: 'database' | 'schema'; /** Configures the ownership of newly created schemas in pg databases. */ role?: string; - /** - * Skip running database migrations. - * NOTE: Currently only supported by the `pg` client. - */ + /** Skip running database migrations. */ skipMigrations?: boolean; /** * Arbitrary config object to pass to knex when initializing @@ -468,10 +465,7 @@ export interface Config { knexConfig?: object; /** Configures the ownership of newly created schemas in pg databases. */ role?: string; - /** - * Skip running database migrations. - * NOTE: Currently only supported by the `pg` client. - */ + /** Skip running database migrations. */ skipMigrations?: boolean; }; }; From f9b1d57da5bff82860480af88b93073c1df81f02 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 27 Aug 2024 10:38:44 +0200 Subject: [PATCH 8/8] chore: Update config Signed-off-by: Johan Haals --- packages/backend-defaults/config.d.ts | 100 ++------------------------ 1 file changed, 4 insertions(+), 96 deletions(-) diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index a13e90e39f..97809da12c 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -375,102 +375,6 @@ export interface Config { }; }; - /** Database connection configuration, select base database type using the `client` field */ - database: { - /** Default database client to use */ - client: 'better-sqlite3' | 'sqlite3' | 'pg'; - /** - * Base database connection string, or object with individual connection properties - * @visibility secret - */ - connection: - | string - | { - /** - * Password that belongs to the client User - * @visibility secret - */ - password?: string; - /** - * Other connection settings - */ - [key: string]: unknown; - }; - /** Database name prefix override */ - prefix?: string; - /** - * Whether to ensure the given database exists by creating it if it does not. - * Defaults to true if unspecified. - */ - ensureExists?: boolean; - /** - * Whether to ensure the given database schema exists by creating it if it does not. - * Defaults to false if unspecified. - * - * NOTE: Currently only supported by the `pg` client when pluginDivisionMode: schema - */ - ensureSchemaExists?: boolean; - /** - * How plugins databases are managed/divided in the provided database instance. - * - * `database` -> Plugins are each given their own database to manage their schemas/tables. - * - * `schema` -> Plugins will be given their own schema (in the specified/default database) - * to manage their tables. - * - * NOTE: Currently only supported by the `pg` client. - * - * @default database - */ - pluginDivisionMode?: 'database' | 'schema'; - /** Configures the ownership of newly created schemas in pg databases. */ - role?: string; - /** Skip running database migrations. */ - skipMigrations?: boolean; - /** - * Arbitrary config object to pass to knex when initializing - * (https://knexjs.org/#Installation-client). Most notable is the debug - * and asyncStackTraces booleans - */ - knexConfig?: object; - /** Plugin specific database configuration and client override */ - plugin?: { - [pluginId: string]: { - /** Database client override */ - client?: 'better-sqlite3' | 'sqlite3' | 'pg'; - /** - * Database connection string or Knex object override - * @visibility secret - */ - connection?: string | object; - /** - * Whether to ensure the given database exists by creating it if it does not. - * Defaults to base config if unspecified. - */ - ensureExists?: boolean; - /** - * Whether to ensure the given database schema exists by creating it if it does not. - * Defaults to false if unspecified. - * - * NOTE: Currently only supported by the `pg` client when pluginDivisionMode: schema - */ - ensureSchemaExists?: boolean; - /** - * Arbitrary config object to pass to knex when initializing - * (https://knexjs.org/#Installation-client). Most notable is the - * debug and asyncStackTraces booleans. - * - * This is merged recursively into the base knexConfig - */ - knexConfig?: object; - /** Configures the ownership of newly created schemas in pg databases. */ - role?: string; - /** Skip running database migrations. */ - skipMigrations?: boolean; - }; - }; - }; - /** * Options used by the default discovery service. */ @@ -549,6 +453,8 @@ export interface Config { * and asyncStackTraces booleans */ knexConfig?: object; + /** Skip running database migrations. */ + skipMigrations?: boolean; /** Plugin specific database configuration and client override */ plugin?: { [pluginId: string]: { @@ -581,6 +487,8 @@ export interface Config { knexConfig?: object; /** Configures the ownership of newly created schemas in pg databases. */ role?: string; + /** Skip running database migrations. */ + skipMigrations?: boolean; }; }; };