Expose types to handle Database management

This implements several types to own and manage databases on a database
server. The current SimpleDatabase* classes preserve the present
behaviour; future implementations can segregate databases to be owned by
different roles.

packages/backend-common/src/database/SingleDatabase.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Knex from 'knex';
+import { Config } from '@backstage/config';
+import { createDatabaseClient, ensureDatabaseExists } from './connection';
+import {
+  DatabaseConfiguration,
+  PluginDatabaseFactory,
+  PluginDatabaseManager,
+} from './types';
+
+export class SingleDatabaseConfiguration implements DatabaseConfiguration {
+  private readonly config: Config;
+
+  constructor(config: Config) {
+    this.config = config;
+  }
+
+  getDatabaseConfig(_pluginId: string): Config {
+    return this.config;
+  }
+
+  getAdminConfig(): Config {
+    return this.config;
+  }
+}
+
+export class SingleDatabaseManager implements PluginDatabaseManager {
+  private readonly config: DatabaseConfiguration;
+
+  constructor(config: DatabaseConfiguration) {
+    this.config = config;
+  }
+
+  getDatabaseFactory(pluginId: string): PluginDatabaseFactory {
+    // eslint-disable-next-line no-use-before-define
+    return new SinglePluginDatabaseFactory(this, pluginId);
+  }
+
+  async getDatabase(pluginId: string, suffix?: string): Promise<Knex> {
+    const config = this.config.getDatabaseConfig(pluginId);
+    const overrides = SingleDatabaseManager.getDatabaseOverrides(
+      pluginId,
+      suffix,
+    );
+    const overrideConfig = overrides.connection as Knex.ConnectionConfig;
+    await this.ensureDatabase(overrideConfig.database);
+
+    return createDatabaseClient(config, overrides);
+  }
+
+  private static getDatabaseOverrides(
+    pluginId: string,
+    suffix?: string,
+  ): Knex.Config {
+    const dbSuffix = suffix ? `_${suffix}` : '';
+    return {
+      connection: {
+        database: `backstage_plugin_${pluginId}${dbSuffix}`,
+      },
+    };
+  }
+
+  private async ensureDatabase(database: string) {
+    const config = this.config.getAdminConfig();
+    await ensureDatabaseExists(config, database);
+  }
+}
+
+export class SinglePluginDatabaseFactory implements PluginDatabaseFactory {
+  private manager: SingleDatabaseManager;
+  private readonly pluginId: string;
+
+  constructor(manager: SingleDatabaseManager, pluginId: string) {
+    this.manager = manager;
+    this.pluginId = pluginId;
+  }
+
+  getDatabase(database?: string): Promise<Knex> {
+    return this.manager.getDatabase(this.pluginId, database);
+  }
+}

packages/backend-common/src/database/index.ts

@@ -15,3 +15,5 @@
  */
 
 export * from './connection';
+export * from './types';
+export * from './SingleDatabase';

packages/backend-common/src/database/postgres.ts

@@ -15,6 +15,7 @@
  */
 
 import knex, { PgConnectionConfig } from 'knex';
+import { cloneDeep } from 'lodash';
 import { Config } from '@backstage/config';
 import { mergeDatabaseConfig } from './config';
 
@@ -44,7 +45,7 @@ export function buildPgDatabaseConfig(
   overrides?: knex.Config,
 ) {
   return mergeDatabaseConfig(
-    dbConfig.get(),
+    cloneDeep(dbConfig.get()),
     {
       connection: getPgConnectionConfig(dbConfig, !!overrides),
       useNullAsDefault: true,

packages/backend-common/src/database/types.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import knex from 'knex';
+import { Config } from '@backstage/config';
+
+/**
+ * DatabaseConfiguration exposes the Application configuration to PluginDatabaseManager instances.
+ */
+export interface DatabaseConfiguration {
+  /**
+   * Returns the database configuration for the given Plugin.
+   */
+  getDatabaseConfig(pluginId: string): Config;
+
+  /**
+   * Returns the database configuration for administrating the database server.
+   */
+  getAdminConfig(): Config;
+}
+
+/**
+ * The PluginDatabaseManager produces PluginDatabaseFactory objects for plugins to consume and store their data.
+ */
+export type PluginDatabaseManager = {
+  /**
+   * getDatabaseFactory returns the database factory object for plugins to obtain database connections.
+   */
+  getDatabaseFactory(pluginId: string): PluginDatabaseFactory;
+};
+
+/**
+ * The PluginDatabaseFactory is used to provide a mechanism for backend plugins to obtain database connections for
+ * itself.
+ *
+ * The purpose of this factory is to allow plugins to get isolated data stores so that plugins are discouraged from database integration.
+ */
+export type PluginDatabaseFactory = {
+  /**
+   * Returns a database connection. Plugins can omit the `database` parameter to get the default plugin database, or
+   * provide an identifier that will be used to identify a separate database from the default.
+   *
+   * Plugins completely own and manage the state of the database (including schema/migrations/data.)
+   */
+  getDatabase(database?: String): Promise<knex>;
+};

packages/backend/src/index.ts

@@ -24,17 +24,17 @@
 
 import Router from 'express-promise-router';
 import {
-  ensureDatabaseExists,
-  createDatabaseClient,
   createServiceBuilder,
   loadBackendConfig,
   getRootLogger,
   useHotMemoize,
   notFoundHandler,
+  SingleDatabaseConfiguration,
+  SingleDatabaseManager,
   SingleHostDiscovery,
   UrlReaders,
 } from '@backstage/backend-common';
-import { ConfigReader, AppConfig } from '@backstage/config';
+import { ConfigReader } from '@backstage/config';
 import healthcheck from './plugins/healthcheck';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
@@ -48,37 +48,29 @@ import graphql from './plugins/graphql';
 import app from './plugins/app';
 import { PluginEnvironment } from './types';
 
-function makeCreateEnv(loadedConfigs: AppConfig[]) {
-  const config = ConfigReader.fromConfigs(loadedConfigs);
+function makeCreateEnv(config: ConfigReader) {
   const root = getRootLogger();
   const reader = UrlReaders.default({ logger: root, config });
   const discovery = SingleHostDiscovery.fromConfig(config);
 
   root.info(`Created UrlReader ${reader}`);
 
+  const databaseConfig = new SingleDatabaseConfiguration(
+    config.getConfig('backend.database'),
+  );
+  const databaseManager = new SingleDatabaseManager(databaseConfig);
+
   return (plugin: string): PluginEnvironment => {
     const logger = root.child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
-      config.getConfig('backend.database'),
-      {
-        connection: {
-          database: `backstage_plugin_${plugin}`,
-        },
-      },
-    );
-    return { logger, database, config, reader, discovery };
+    const databaseFactory = databaseManager.getDatabaseFactory(plugin);
+    return { logger, database: databaseFactory, config, reader, discovery };
   };
 }
 
 async function main() {
   const configs = await loadBackendConfig();
   const configReader = ConfigReader.fromConfigs(configs);
-  const createEnv = makeCreateEnv(configs);
-  await ensureDatabaseExists(
-    configReader.getConfig('backend.database'),
-    'backstage_plugin_catalog',
-    'backstage_plugin_auth',
-  );
+  const createEnv = makeCreateEnv(configReader);
 
   const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck'));
   const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));

packages/backend/src/plugins/catalog.ts

@@ -34,7 +34,10 @@ export default async function createPlugin({
 }: PluginEnvironment) {
   const locationReader = new LocationReaders({ logger, reader, config });
 
-  const db = await DatabaseManager.createDatabase(database, { logger });
+  const db = await DatabaseManager.createDatabase(
+    await database.getDatabase(),
+    { logger },
+  );
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);
   const locationsCatalog = new DatabaseLocationsCatalog(db);
   const higherOrderOperation = new HigherOrderOperations(

packages/backend/src/types.ts

@@ -14,14 +14,17 @@
  * limitations under the License.
  */
 
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
+import {
+  PluginDatabaseFactory,
+  PluginEndpointDiscovery,
+  UrlReader
+} from '@backstage/backend-common';
 
 export type PluginEnvironment = {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseFactory;
   config: Config;
   reader: UrlReader;
   discovery: PluginEndpointDiscovery;

packages/create-app/templates/default-app/packages/backend/src/index.ts

@@ -9,16 +9,17 @@
 import Router from 'express-promise-router';
 import {
   ensureDatabaseExists,
-  createDatabaseClient,
   createServiceBuilder,
   loadBackendConfig,
   getRootLogger,
   useHotMemoize,
   notFoundHandler,
+  SingleDatabaseConfiguration,
+  SingleDatabaseManager,
   SingleHostDiscovery,
   UrlReaders,
 } from '@backstage/backend-common';
-import { ConfigReader, AppConfig } from '@backstage/config';
+import { ConfigReader } from '@backstage/config';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
 import scaffolder from './plugins/scaffolder';
@@ -26,32 +27,29 @@ import proxy from './plugins/proxy';
 import techdocs from './plugins/techdocs';
 import { PluginEnvironment } from './types';
 
-function makeCreateEnv(loadedConfigs: AppConfig[]) {
-  const config = ConfigReader.fromConfigs(loadedConfigs);
+function makeCreateEnv(config: ConfigReader) {
   const root = getRootLogger();
   const reader = UrlReaders.default({ logger: root, config });
   const discovery = SingleHostDiscovery.fromConfig(config);
 
   root.info(`Created UrlReader ${reader}`);
 
+  const databaseConfig = new SingleDatabaseConfiguration(
+    config.getConfig('backend.database'),
+  )
+  const databaseManager = new SingleDatabaseManager(databaseConfig);
+
   return (plugin: string): PluginEnvironment => {
     const logger = root.child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
-      config.getConfig('backend.database'),
-      {
-        connection: {
-          database: `backstage_plugin_${plugin}`,
-        },
-      },
-    );
-    return { logger, database, config, reader, discovery };
+    const databaseFactory = databaseManager.getDatabaseFactory(plugin);
+    return { logger, database: databaseFactory, config, reader, discovery };
   };
 }
 
 async function main() {
   const configs = await loadBackendConfig();
   const configReader = ConfigReader.fromConfigs(configs);
-  const createEnv = makeCreateEnv(configs);
+  const createEnv = makeCreateEnv(configReader);
   await ensureDatabaseExists(
     configReader.getConfig('backend.database'),
     'backstage_plugin_catalog',

plugins/auth-backend/src/service/router.ts

@@ -17,7 +17,6 @@
 import express from 'express';
 import Router from 'express-promise-router';
 import cookieParser from 'cookie-parser';
-import Knex from 'knex';
 import { Logger } from 'winston';
 import { createAuthProvider } from '../providers';
 import { Config } from '@backstage/config';
@@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity';
 import {
   NotFoundError,
   PluginEndpointDiscovery,
+  PluginDatabaseFactory,
 } from '@backstage/backend-common';
 
 export interface RouterOptions {
   logger: Logger;
-  database: Knex;
+  database: PluginDatabaseFactory;
   config: Config;
   discovery: PluginEndpointDiscovery;
 }
@@ -47,7 +47,9 @@ export async function createRouter({
 
   const keyDurationSeconds = 3600;
 
-  const keyStore = await DatabaseKeyStore.create({ database });
+  const keyStore = await DatabaseKeyStore.create({
+    database: await database.getDatabase(),
+  });
   const tokenIssuer = new TokenFactory({
     issuer: authUrl,
     keyStore,

plugins/auth-backend/src/service/standaloneServer.ts

@@ -53,7 +53,11 @@ export async function startStandaloneServer(
   const router = await createRouter({
     logger,
     config,
-    database,
+    database: {
+      async getDatabase(_database?: string) {
+        return database;
+      },
+    },
     discovery,
   });