Merge pull request #5308 from SDA-SE/feat/sqlite-dir

Support configuration of file storage for SQLite databases

.changeset/six-turtles-sip.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+---
+
+Support configuration of file storage for SQLite databases. Every plugin has its
+own database file at the specified path.

packages/backend-common/config.d.ts

@@ -57,7 +57,7 @@ export interface Config {
     database:
       | {
           client: 'sqlite3';
-          connection: ':memory:' | string;
+          connection: ':memory:' | string | { filename: string };
         }
       | {
           client: 'pg';

packages/backend-common/src/database/connection.ts

@@ -37,7 +37,7 @@ export function createDatabaseClient(
   if (client === 'pg') {
     return createPgDatabaseClient(dbConfig, overrides);
   } else if (client === 'sqlite3') {
-    return createSqliteDatabaseClient(dbConfig);
+    return createSqliteDatabaseClient(dbConfig, overrides);
   }
 
   return knexFactory(mergeDatabaseConfig(dbConfig.get(), overrides));

packages/backend-common/src/database/sqlite3.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
+import path from 'path';
 import {
   buildSqliteDatabaseConfig,
   createSqliteDatabaseClient,
@@ -25,25 +26,70 @@ describe('sqlite3', () => {
     new ConfigReader({ client: 'sqlite3', connection });
 
   describe('buildSqliteDatabaseConfig', () => {
-    it('buidls a string connection', () => {
+    it('builds an in-memory connection', () => {
       expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({
         client: 'sqlite3',
-        connection: ':memory:',
+        connection: { filename: ':memory:' },
         useNullAsDefault: true,
       });
     });
 
-    it('builds a filename connection', () => {
+    it('builds an in-memory connection by override with filename', () => {
+      expect(
+        buildSqliteDatabaseConfig(
+          createConfig(path.join('path', 'to', 'foo')),
+          { connection: ':memory:' },
+        ),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: { filename: ':memory:' },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection, normalize config with filename', () => {
+      expect(
+        buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: { filename: path.join('path', 'to', 'foo') },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection', () => {
       expect(
         buildSqliteDatabaseConfig(
           createConfig({
-            filename: '/path/to/foo',
+            filename: path.join('path', 'to', 'foo'),
           }),
         ),
       ).toEqual({
         client: 'sqlite3',
         connection: {
-          filename: '/path/to/foo',
+          filename: path.join('path', 'to', 'foo'),
+        },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a persistent connection per database', () => {
+      expect(
+        buildSqliteDatabaseConfig(
+          createConfig({
+            filename: path.join('path', 'to', 'foo'),
+          }),
+          {
+            connection: {
+              database: 'my-database',
+            },
+          },
+        ),
+      ).toEqual({
+        client: 'sqlite3',
+        connection: {
+          filename: path.join('path', 'to', 'foo', 'my-database.sqlite'),
+          database: 'my-database',
         },
         useNullAsDefault: true,
       });
@@ -52,12 +98,12 @@ describe('sqlite3', () => {
     it('replaces the connection with an override', () => {
       expect(
         buildSqliteDatabaseConfig(createConfig(':memory:'), {
-          connection: { filename: '/path/to/foo' },
+          connection: { filename: path.join('path', 'to', 'foo') },
         }),
       ).toEqual({
         client: 'sqlite3',
         connection: {
-          filename: '/path/to/foo',
+          filename: path.join('path', 'to', 'foo'),
         },
         useNullAsDefault: true,
       });

packages/backend-common/src/database/sqlite3.ts

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
-import knexFactory, { Knex } from 'knex';
 import { Config } from '@backstage/config';
+import { ensureDirSync } from 'fs-extra';
+import knexFactory, { Knex } from 'knex';
+import path from 'path';
 import { mergeDatabaseConfig } from './config';
 
 /**
@@ -29,6 +31,19 @@ export function createSqliteDatabaseClient(
   overrides?: Knex.Config,
 ) {
   const knexConfig = buildSqliteDatabaseConfig(dbConfig, overrides);
+
+  // If storage on disk is used, ensure that the directory exists
+  if (
+    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename &&
+    (knexConfig.connection as Knex.Sqlite3ConnectionConfig).filename !==
+      ':memory:'
+  ) {
+    const { filename } = knexConfig.connection as Knex.Sqlite3ConnectionConfig;
+    const directory = path.dirname(filename);
+
+    ensureDirSync(directory);
+  }
+
   const database = knexFactory(knexConfig);
 
   database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
@@ -47,12 +62,40 @@ export function createSqliteDatabaseClient(
 export function buildSqliteDatabaseConfig(
   dbConfig: Config,
   overrides?: Knex.Config,
-) {
-  return mergeDatabaseConfig(
-    dbConfig.get(),
+): Knex.Config {
+  const baseConfig = dbConfig.get<Knex.Config>();
+
+  // Normalize config to always contain a connection object
+  if (typeof baseConfig.connection === 'string') {
+    baseConfig.connection = { filename: baseConfig.connection };
+  }
+  if (overrides && typeof overrides.connection === 'string') {
+    overrides.connection = { filename: overrides.connection };
+  }
+
+  const config: Knex.Config = mergeDatabaseConfig(
+    {
+      connection: {},
+    },
+    baseConfig,
     {
       useNullAsDefault: true,
     },
     overrides,
   );
+
+  // If we don't create an in-memory database, interpret the connection string
+  // as a directory that contains multiple sqlite files based on the database
+  // name.
+  const database = (config.connection as Knex.ConnectionConfig).database;
+  const sqliteConnection = config.connection as Knex.Sqlite3ConnectionConfig;
+
+  if (database && sqliteConnection.filename !== ':memory:') {
+    sqliteConnection.filename = path.join(
+      sqliteConnection.filename,
+      `${database}.sqlite`,
+    );
+  }
+
+  return config;
 }