From b42531cfedc94bbdc4fa8a445f4a2b2a341bd395 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 13 Apr 2021 13:53:43 +0200 Subject: [PATCH 1/6] Support configuration of file storage for SQLite databases Signed-off-by: Oliver Sand --- .changeset/six-turtles-sip.md | 6 +++ packages/backend-common/config.d.ts | 2 +- .../backend-common/src/database/connection.ts | 2 +- .../src/database/sqlite3.test.ts | 36 +++++++++++++-- .../backend-common/src/database/sqlite3.ts | 45 +++++++++++++++++-- 5 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 .changeset/six-turtles-sip.md diff --git a/.changeset/six-turtles-sip.md b/.changeset/six-turtles-sip.md new file mode 100644 index 0000000000..e71b04e64d --- /dev/null +++ b/.changeset/six-turtles-sip.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +--- + +Support configuration of file storage for SQLite databases. Every plugin has its +own database file at the specified path. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 74c199e737..41845f952e 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -57,7 +57,7 @@ export interface Config { database: | { client: 'sqlite3'; - connection: ':memory:' | string; + connection: ':memory:' | string | { filename: string }; } | { client: 'pg'; diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 17ef2c461d..3502e21674 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -37,7 +37,7 @@ export function createDatabaseClient( if (client === 'pg') { return createPgDatabaseClient(dbConfig, overrides); } else if (client === 'sqlite3') { - return createSqliteDatabaseClient(dbConfig); + return createSqliteDatabaseClient(dbConfig, overrides); } return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides)); diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/sqlite3.test.ts index a6b8e5d84d..3066cb6c9d 100644 --- a/packages/backend-common/src/database/sqlite3.test.ts +++ b/packages/backend-common/src/database/sqlite3.test.ts @@ -25,15 +25,23 @@ describe('sqlite3', () => { new ConfigReader({ client: 'sqlite3', connection }); describe('buildSqliteDatabaseConfig', () => { - it('buidls a string connection', () => { + it('builds an in memory connection', () => { expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({ client: 'sqlite3', - connection: ':memory:', + connection: { filename: ':memory:' }, useNullAsDefault: true, }); }); - it('builds a filename connection', () => { + it('builds a persistent connection, normalize config with filename', () => { + expect(buildSqliteDatabaseConfig(createConfig('/path/to/foo'))).toEqual({ + client: 'sqlite3', + connection: { filename: '/path/to/foo' }, + useNullAsDefault: true, + }); + }); + + it('builds a persistent connection', () => { expect( buildSqliteDatabaseConfig( createConfig({ @@ -49,6 +57,28 @@ describe('sqlite3', () => { }); }); + it('builds a persistent connection per database', () => { + expect( + buildSqliteDatabaseConfig( + createConfig({ + filename: '/path/to/foo', + }), + { + connection: { + database: 'my-database', + }, + }, + ), + ).toEqual({ + client: 'sqlite3', + connection: { + filename: '/path/to/foo/my-database.sqlite', + database: 'my-database', + }, + useNullAsDefault: true, + }); + }); + it('replaces the connection with an override', () => { expect( buildSqliteDatabaseConfig(createConfig(':memory:'), { diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index f5742c68a0..9250315413 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -14,8 +14,10 @@ * limitations under the License. */ -import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; +import fs from 'fs'; +import knexFactory, { Knex } from 'knex'; +import path from 'path'; import { mergeDatabaseConfig } from './config'; /** @@ -29,6 +31,20 @@ export function createSqliteDatabaseClient( overrides?: Knex.Config, ) { const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); + + // If storage on disk is used, ensure that the directory exists + if ( + typeof knexConfig.connection === 'object' && + (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename + ) { + const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; + const directory = path.dirname(filename); + + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } + } + const database = knexFactory(knexConfig); database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { @@ -47,12 +63,33 @@ export function createSqliteDatabaseClient( export function buildSqliteDatabaseConfig( dbConfig: Config, overrides?: Knex.Config, -) { - return mergeDatabaseConfig( - dbConfig.get(), +): Knex.Config { + const baseConfig = dbConfig.get(); + + // Normalize config to always contain a connection object + if (typeof baseConfig.connection === 'string') { + baseConfig.connection = { filename: baseConfig.connection }; + } + + const config: Knex.Config = mergeDatabaseConfig( + baseConfig, { useNullAsDefault: true, }, overrides, ); + + // If we don't create an in-memory database, interpret the connection string + // as a directory that contains multiple sqlite files based on the database + // name. + if (config.connection && typeof config.connection === 'object') { + const database = (config.connection as Knex.ConnectionConfig).database; + const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; + + if (database && sqliteConnection.filename !== ':memory:') { + sqliteConnection.filename = `${sqliteConnection.filename}/${database}.sqlite`; + } + } + + return config; } From 2fc0c9b1c0d731f45a6b301c3848123161dff244 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 14 Apr 2021 15:41:27 +0200 Subject: [PATCH 2/6] Make tests path separator independent and work on Windows Signed-off-by: Oliver Sand --- .../src/database/sqlite3.test.ts | 19 +++++++++++-------- .../backend-common/src/database/sqlite3.ts | 5 ++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/sqlite3.test.ts index 3066cb6c9d..cf22fa28ee 100644 --- a/packages/backend-common/src/database/sqlite3.test.ts +++ b/packages/backend-common/src/database/sqlite3.test.ts @@ -15,6 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; +import path from 'path'; import { buildSqliteDatabaseConfig, createSqliteDatabaseClient, @@ -34,9 +35,11 @@ describe('sqlite3', () => { }); it('builds a persistent connection, normalize config with filename', () => { - expect(buildSqliteDatabaseConfig(createConfig('/path/to/foo'))).toEqual({ + expect( + buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))), + ).toEqual({ client: 'sqlite3', - connection: { filename: '/path/to/foo' }, + connection: { filename: path.join('path', 'to', 'foo') }, useNullAsDefault: true, }); }); @@ -45,13 +48,13 @@ describe('sqlite3', () => { expect( buildSqliteDatabaseConfig( createConfig({ - filename: '/path/to/foo', + filename: path.join('path', 'to', 'foo'), }), ), ).toEqual({ client: 'sqlite3', connection: { - filename: '/path/to/foo', + filename: path.join('path', 'to', 'foo'), }, useNullAsDefault: true, }); @@ -61,7 +64,7 @@ describe('sqlite3', () => { expect( buildSqliteDatabaseConfig( createConfig({ - filename: '/path/to/foo', + filename: path.join('path', 'to', 'foo'), }), { connection: { @@ -72,7 +75,7 @@ describe('sqlite3', () => { ).toEqual({ client: 'sqlite3', connection: { - filename: '/path/to/foo/my-database.sqlite', + filename: path.join('path', 'to', 'foo', 'my-database.sqlite'), database: 'my-database', }, useNullAsDefault: true, @@ -82,12 +85,12 @@ describe('sqlite3', () => { it('replaces the connection with an override', () => { expect( buildSqliteDatabaseConfig(createConfig(':memory:'), { - connection: { filename: '/path/to/foo' }, + connection: { filename: path.join('path', 'to', 'foo') }, }), ).toEqual({ client: 'sqlite3', connection: { - filename: '/path/to/foo', + filename: path.join('path', 'to', 'foo'), }, useNullAsDefault: true, }); diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index 9250315413..c73f3f7029 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -87,7 +87,10 @@ export function buildSqliteDatabaseConfig( const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; if (database && sqliteConnection.filename !== ':memory:') { - sqliteConnection.filename = `${sqliteConnection.filename}/${database}.sqlite`; + sqliteConnection.filename = path.join( + sqliteConnection.filename, + `${database}.sqlite`, + ); } } From 4b6e31ece2ad0233f010176fdade5e85d605665f Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 14 Apr 2021 15:46:35 +0200 Subject: [PATCH 3/6] Also normalize the config override Signed-off-by: Oliver Sand --- .../backend-common/src/database/sqlite3.test.ts | 15 ++++++++++++++- packages/backend-common/src/database/sqlite3.ts | 7 +++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/sqlite3.test.ts index cf22fa28ee..86f3a6968b 100644 --- a/packages/backend-common/src/database/sqlite3.test.ts +++ b/packages/backend-common/src/database/sqlite3.test.ts @@ -26,7 +26,7 @@ describe('sqlite3', () => { new ConfigReader({ client: 'sqlite3', connection }); describe('buildSqliteDatabaseConfig', () => { - it('builds an in memory connection', () => { + it('builds an in-memory connection', () => { expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({ client: 'sqlite3', connection: { filename: ':memory:' }, @@ -34,6 +34,19 @@ describe('sqlite3', () => { }); }); + it('builds an in-memory connection by override with filename', () => { + expect( + buildSqliteDatabaseConfig( + createConfig(path.join('path', 'to', 'foo')), + { connection: ':memory:' }, + ), + ).toEqual({ + client: 'sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + }); + }); + it('builds a persistent connection, normalize config with filename', () => { expect( buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))), diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index c73f3f7029..170ea5215d 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -34,7 +34,7 @@ export function createSqliteDatabaseClient( // If storage on disk is used, ensure that the directory exists if ( - typeof knexConfig.connection === 'object' && + knexConfig.connection && (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename ) { const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; @@ -70,6 +70,9 @@ export function buildSqliteDatabaseConfig( if (typeof baseConfig.connection === 'string') { baseConfig.connection = { filename: baseConfig.connection }; } + if (overrides && typeof overrides.connection === 'string') { + overrides.connection = { filename: overrides.connection }; + } const config: Knex.Config = mergeDatabaseConfig( baseConfig, @@ -82,7 +85,7 @@ export function buildSqliteDatabaseConfig( // If we don't create an in-memory database, interpret the connection string // as a directory that contains multiple sqlite files based on the database // name. - if (config.connection && typeof config.connection === 'object') { + if (config.connection) { const database = (config.connection as Knex.ConnectionConfig).database; const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; From 0c49f4461ef0cec425e4282e6666ecd2327a5f9b Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 14 Apr 2021 15:56:49 +0200 Subject: [PATCH 4/6] Ensure that the connection object is always initialized Signed-off-by: Oliver Sand --- .../backend-common/src/database/sqlite3.ts | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index 170ea5215d..d0d36df51d 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -33,10 +33,7 @@ export function createSqliteDatabaseClient( const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); // If storage on disk is used, ensure that the directory exists - if ( - knexConfig.connection && - (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename - ) { + if ((knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename) { const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; const directory = path.dirname(filename); @@ -75,6 +72,9 @@ export function buildSqliteDatabaseConfig( } const config: Knex.Config = mergeDatabaseConfig( + { + connection: {}, + }, baseConfig, { useNullAsDefault: true, @@ -85,16 +85,14 @@ export function buildSqliteDatabaseConfig( // If we don't create an in-memory database, interpret the connection string // as a directory that contains multiple sqlite files based on the database // name. - if (config.connection) { - const database = (config.connection as Knex.ConnectionConfig).database; - const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; + const database = (config.connection as Knex.ConnectionConfig).database; + const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig; - if (database && sqliteConnection.filename !== ':memory:') { - sqliteConnection.filename = path.join( - sqliteConnection.filename, - `${database}.sqlite`, - ); - } + if (database && sqliteConnection.filename !== ':memory:') { + sqliteConnection.filename = path.join( + sqliteConnection.filename, + `${database}.sqlite`, + ); } return config; From 00755782242b024ef29e200216db0fd477d9c7d3 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 15 Apr 2021 10:55:43 +0200 Subject: [PATCH 5/6] Use ensureDirSync Signed-off-by: Oliver Sand --- packages/backend-common/src/database/sqlite3.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index d0d36df51d..90328f9527 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -15,7 +15,7 @@ */ import { Config } from '@backstage/config'; -import fs from 'fs'; +import { ensureDirSync } from 'fs-extra'; import knexFactory, { Knex } from 'knex'; import path from 'path'; import { mergeDatabaseConfig } from './config'; @@ -37,9 +37,7 @@ export function createSqliteDatabaseClient( const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; const directory = path.dirname(filename); - if (!fs.existsSync(directory)) { - fs.mkdirSync(directory, { recursive: true }); - } + ensureDirSync(directory); } const database = knexFactory(knexConfig); From 3f048fbd250aca3cb68ae489f87ee4c714659d76 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 15 Apr 2021 11:30:12 +0200 Subject: [PATCH 6/6] Don't ensure directory exists if :memory: storage is used Signed-off-by: Oliver Sand --- packages/backend-common/src/database/sqlite3.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts index 90328f9527..d4169e3899 100644 --- a/packages/backend-common/src/database/sqlite3.ts +++ b/packages/backend-common/src/database/sqlite3.ts @@ -33,7 +33,11 @@ export function createSqliteDatabaseClient( const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides); // If storage on disk is used, ensure that the directory exists - if ((knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename) { + if ( + (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename && + (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !== + ':memory:' + ) { const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig; const directory = path.dirname(filename);