Support configuration of file storage for SQLite databases

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

.changeset/six-turtles-sip.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+---
+
+Support configuration of file storage for SQLite databases. Every plugin has its
+own database file at the specified path.

packages/backend-common/config.d.ts

@@ -57,7 +57,7 @@ export interface Config {
     database:
       | {
           client: 'sqlite3';
-          connection: ':memory:' | string;
+          connection: ':memory:' | string | { filename: string };
         }
       | {
           client: 'pg';

packages/backend-common/src/database/connection.ts

@@ -37,7 +37,7 @@ export function createDatabaseClient(
   if (client === 'pg') {
     return createPgDatabaseClient(dbConfig, overrides);
   } else if (client === 'sqlite3') {
-    return createSqliteDatabaseClient(dbConfig);
+    return createSqliteDatabaseClient(dbConfig, overrides);
   }
 
   return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides));

packages/backend-common/src/database/sqlite3.test.ts

@@ -25,15 +25,23 @@ describe('sqlite3', () => {
     new ConfigReader({ client: 'sqlite3', connection });
 
   describe('buildSqliteDatabaseConfig', () => {
-    it('buidls a string connection', () => {
+    it('builds an in memory connection', () => {
       expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({
         client: 'sqlite3',
-        connection: ':memory:',
+        connection: { filename: ':memory:' },
         useNullAsDefault: true,
       });
     });
 
-    it('builds a filename connection', () => {
+    it('builds a persistent connection, normalize config with filename', () => {
+      expect(buildSqliteDatabaseConfig(createConfig('/path/to/foo'))).toEqual({
+        client: 'sqlite3',
+        connection: { filename: '/path/to/foo' },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection', () => {
       expect(
         buildSqliteDatabaseConfig(
           createConfig({
@@ -49,6 +57,28 @@ describe('sqlite3', () => {
       });
     });
 
+    it('builds a persistent connection per database', () => {
+      expect(
+        buildSqliteDatabaseConfig(
+          createConfig({
+            filename: '/path/to/foo',
+          }),
+          {
+            connection: {
+              database: 'my-database',
+            },
+          },
+        ),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: {
+          filename: '/path/to/foo/my-database.sqlite',
+          database: 'my-database',
+        },
+        useNullAsDefault: true,
+      });
+    });
+
     it('replaces the connection with an override', () => {
       expect(
         buildSqliteDatabaseConfig(createConfig(':memory:'), {

packages/backend-common/src/database/sqlite3.ts

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
-import knexFactory, { Knex } from 'knex';
 import { Config } from '@backstage/config';
+import fs from 'fs';
+import knexFactory, { Knex } from 'knex';
+import path from 'path';
 import { mergeDatabaseConfig } from './config';
 
 /**
@@ -29,6 +31,20 @@ export function createSqliteDatabaseClient(
   overrides?: Knex.Config,
 ) {
   const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides);
+
+  // If storage on disk is used, ensure that the directory exists
+  if (
+    typeof knexConfig.connection === 'object' &&
+    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename
+  ) {
+    const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
+    const directory = path.dirname(filename);
+
+    if (!fs.existsSync(directory)) {
+      fs.mkdirSync(directory, { recursive: true });
+    }
+  }
+
   const database = knexFactory(knexConfig);
 
   database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
@@ -47,12 +63,33 @@ export function createSqliteDatabaseClient(
 export function buildSqliteDatabaseConfig(
   dbConfig: Config,
   overrides?: Knex.Config,
-) {
-  return mergeDatabaseConfig(
-    dbConfig.get(),
+): Knex.Config {
+  const baseConfig = dbConfig.get<Knex.Config>();
+
+  // Normalize config to always contain a connection object
+  if (typeof baseConfig.connection === 'string') {
+    baseConfig.connection = { filename: baseConfig.connection };
+  }
+
+  const config: Knex.Config = mergeDatabaseConfig(
+    baseConfig,
     {
       useNullAsDefault: true,
     },
     overrides,
   );
+
+  // If we don't create an in-memory database, interpret the connection string
+  // as a directory that contains multiple sqlite files based on the database
+  // name.
+  if (config.connection && typeof config.connection === 'object') {
+    const database = (config.connection as Knex.ConnectionConfig).database;
+    const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig;
+
+    if (database && sqliteConnection.filename !== ':memory:') {
+      sqliteConnection.filename = `${sqliteConnection.filename}/${database}.sqlite`;
+    }
+  }
+
+  return config;
 }