From e277c5dfa587134672ac3ba81351dee03deb8022 Mon Sep 17 00:00:00 2001 From: Joel Low Date: Thu, 1 Oct 2020 17:38:12 +0800 Subject: [PATCH] Expose types to handle Database management This implements several types to own and manage databases on a database server. The current SimpleDatabase* classes preserve the present behaviour; future implementations can segregate databases to be owned by different roles. --- .../src/database/SingleDatabase.ts | 96 +++++++++++++++++++ packages/backend-common/src/database/index.ts | 2 + .../backend-common/src/database/postgres.ts | 3 +- packages/backend-common/src/database/types.ts | 59 ++++++++++++ packages/backend/src/index.ts | 32 +++---- packages/backend/src/plugins/catalog.ts | 5 +- packages/backend/src/types.ts | 9 +- .../default-app/packages/backend/src/index.ts | 26 +++-- plugins/auth-backend/src/service/router.ts | 8 +- .../src/service/standaloneServer.ts | 6 +- 10 files changed, 203 insertions(+), 43 deletions(-) create mode 100644 packages/backend-common/src/database/SingleDatabase.ts create mode 100644 packages/backend-common/src/database/types.ts diff --git a/packages/backend-common/src/database/SingleDatabase.ts b/packages/backend-common/src/database/SingleDatabase.ts new file mode 100644 index 0000000000..b599232ee8 --- /dev/null +++ b/packages/backend-common/src/database/SingleDatabase.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Knex from 'knex'; +import { Config } from '@backstage/config'; +import { createDatabaseClient, ensureDatabaseExists } from './connection'; +import { + DatabaseConfiguration, + PluginDatabaseFactory, + PluginDatabaseManager, +} from './types'; + +export class SingleDatabaseConfiguration implements DatabaseConfiguration { + private readonly config: Config; + + constructor(config: Config) { + this.config = config; + } + + getDatabaseConfig(_pluginId: string): Config { + return this.config; + } + + getAdminConfig(): Config { + return this.config; + } +} + +export class SingleDatabaseManager implements PluginDatabaseManager { + private readonly config: DatabaseConfiguration; + + constructor(config: DatabaseConfiguration) { + this.config = config; + } + + getDatabaseFactory(pluginId: string): PluginDatabaseFactory { + // eslint-disable-next-line no-use-before-define + return new SinglePluginDatabaseFactory(this, pluginId); + } + + async getDatabase(pluginId: string, suffix?: string): Promise { + const config = this.config.getDatabaseConfig(pluginId); + const overrides = SingleDatabaseManager.getDatabaseOverrides( + pluginId, + suffix, + ); + const overrideConfig = overrides.connection as Knex.ConnectionConfig; + await this.ensureDatabase(overrideConfig.database); + + return createDatabaseClient(config, overrides); + } + + private static getDatabaseOverrides( + pluginId: string, + suffix?: string, + ): Knex.Config { + const dbSuffix = suffix ? `_${suffix}` : ''; + return { + connection: { + database: `backstage_plugin_${pluginId}${dbSuffix}`, + }, + }; + } + + private async ensureDatabase(database: string) { + const config = this.config.getAdminConfig(); + await ensureDatabaseExists(config, database); + } +} + +export class SinglePluginDatabaseFactory implements PluginDatabaseFactory { + private manager: SingleDatabaseManager; + private readonly pluginId: string; + + constructor(manager: SingleDatabaseManager, pluginId: string) { + this.manager = manager; + this.pluginId = pluginId; + } + + getDatabase(database?: string): Promise { + return this.manager.getDatabase(this.pluginId, database); + } +} diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 38d3d6224b..c667edc659 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -15,3 +15,5 @@ */ export * from './connection'; +export * from './types'; +export * from './SingleDatabase'; diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 82e971f081..9972377de8 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -15,6 +15,7 @@ */ import knex, { PgConnectionConfig } from 'knex'; +import { cloneDeep } from 'lodash'; import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; @@ -44,7 +45,7 @@ export function buildPgDatabaseConfig( overrides?: knex.Config, ) { return mergeDatabaseConfig( - dbConfig.get(), + cloneDeep(dbConfig.get()), { connection: getPgConnectionConfig(dbConfig, !!overrides), useNullAsDefault: true, diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts new file mode 100644 index 0000000000..00ee82f22a --- /dev/null +++ b/packages/backend-common/src/database/types.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; +import { Config } from '@backstage/config'; + +/** + * DatabaseConfiguration exposes the Application configuration to PluginDatabaseManager instances. + */ +export interface DatabaseConfiguration { + /** + * Returns the database configuration for the given Plugin. + */ + getDatabaseConfig(pluginId: string): Config; + + /** + * Returns the database configuration for administrating the database server. + */ + getAdminConfig(): Config; +} + +/** + * The PluginDatabaseManager produces PluginDatabaseFactory objects for plugins to consume and store their data. + */ +export type PluginDatabaseManager = { + /** + * getDatabaseFactory returns the database factory object for plugins to obtain database connections. + */ + getDatabaseFactory(pluginId: string): PluginDatabaseFactory; +}; + +/** + * The PluginDatabaseFactory is used to provide a mechanism for backend plugins to obtain database connections for + * itself. + * + * The purpose of this factory is to allow plugins to get isolated data stores so that plugins are discouraged from database integration. + */ +export type PluginDatabaseFactory = { + /** + * Returns a database connection. Plugins can omit the `database` parameter to get the default plugin database, or + * provide an identifier that will be used to identify a separate database from the default. + * + * Plugins completely own and manage the state of the database (including schema/migrations/data.) + */ + getDatabase(database?: String): Promise; +}; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 959af71427..192ceb95a2 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -24,17 +24,17 @@ import Router from 'express-promise-router'; import { - ensureDatabaseExists, - createDatabaseClient, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, notFoundHandler, + SingleDatabaseConfiguration, + SingleDatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; -import { ConfigReader, AppConfig } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; @@ -48,37 +48,29 @@ import graphql from './plugins/graphql'; import app from './plugins/app'; import { PluginEnvironment } from './types'; -function makeCreateEnv(loadedConfigs: AppConfig[]) { - const config = ConfigReader.fromConfigs(loadedConfigs); +function makeCreateEnv(config: ConfigReader) { const root = getRootLogger(); const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); + const databaseConfig = new SingleDatabaseConfiguration( + config.getConfig('backend.database'), + ); + const databaseManager = new SingleDatabaseManager(databaseConfig); + return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); - const database = createDatabaseClient( - config.getConfig('backend.database'), - { - connection: { - database: `backstage_plugin_${plugin}`, - }, - }, - ); - return { logger, database, config, reader, discovery }; + const databaseFactory = databaseManager.getDatabaseFactory(plugin); + return { logger, database: databaseFactory, config, reader, discovery }; }; } async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); - const createEnv = makeCreateEnv(configs); - await ensureDatabaseExists( - configReader.getConfig('backend.database'), - 'backstage_plugin_catalog', - 'backstage_plugin_auth', - ); + const createEnv = makeCreateEnv(configReader); const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index c1d390f8cc..9ab2203ca2 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -34,7 +34,10 @@ export default async function createPlugin({ }: PluginEnvironment) { const locationReader = new LocationReaders({ logger, reader, config }); - const db = await DatabaseManager.createDatabase(database, { logger }); + const db = await DatabaseManager.createDatabase( + await database.getDatabase(), + { logger }, + ); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); const higherOrderOperation = new HigherOrderOperations( diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 9257fcc9cf..98797eddf9 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -14,14 +14,17 @@ * limitations under the License. */ -import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; +import { + PluginDatabaseFactory, + PluginEndpointDiscovery, + UrlReader +} from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; - database: Knex; + database: PluginDatabaseFactory; config: Config; reader: UrlReader; discovery: PluginEndpointDiscovery; diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 684db8aab8..437ef5afcd 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -9,16 +9,17 @@ import Router from 'express-promise-router'; import { ensureDatabaseExists, - createDatabaseClient, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, notFoundHandler, + SingleDatabaseConfiguration, + SingleDatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; -import { ConfigReader, AppConfig } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import scaffolder from './plugins/scaffolder'; @@ -26,32 +27,29 @@ import proxy from './plugins/proxy'; import techdocs from './plugins/techdocs'; import { PluginEnvironment } from './types'; -function makeCreateEnv(loadedConfigs: AppConfig[]) { - const config = ConfigReader.fromConfigs(loadedConfigs); +function makeCreateEnv(config: ConfigReader) { const root = getRootLogger(); const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); + const databaseConfig = new SingleDatabaseConfiguration( + config.getConfig('backend.database'), + ) + const databaseManager = new SingleDatabaseManager(databaseConfig); + return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); - const database = createDatabaseClient( - config.getConfig('backend.database'), - { - connection: { - database: `backstage_plugin_${plugin}`, - }, - }, - ); - return { logger, database, config, reader, discovery }; + const databaseFactory = databaseManager.getDatabaseFactory(plugin); + return { logger, database: databaseFactory, config, reader, discovery }; }; } async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); - const createEnv = makeCreateEnv(configs); + const createEnv = makeCreateEnv(configReader); await ensureDatabaseExists( configReader.getConfig('backend.database'), 'backstage_plugin_catalog', diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index bcd1fc1794..d514586ec9 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -17,7 +17,6 @@ import express from 'express'; import Router from 'express-promise-router'; import cookieParser from 'cookie-parser'; -import Knex from 'knex'; import { Logger } from 'winston'; import { createAuthProvider } from '../providers'; import { Config } from '@backstage/config'; @@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity'; import { NotFoundError, PluginEndpointDiscovery, + PluginDatabaseFactory, } from '@backstage/backend-common'; export interface RouterOptions { logger: Logger; - database: Knex; + database: PluginDatabaseFactory; config: Config; discovery: PluginEndpointDiscovery; } @@ -47,7 +47,9 @@ export async function createRouter({ const keyDurationSeconds = 3600; - const keyStore = await DatabaseKeyStore.create({ database }); + const keyStore = await DatabaseKeyStore.create({ + database: await database.getDatabase(), + }); const tokenIssuer = new TokenFactory({ issuer: authUrl, keyStore, diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts index 71771340a5..e51b959188 100644 --- a/plugins/auth-backend/src/service/standaloneServer.ts +++ b/plugins/auth-backend/src/service/standaloneServer.ts @@ -53,7 +53,11 @@ export async function startStandaloneServer( const router = await createRouter({ logger, config, - database, + database: { + async getDatabase(_database?: string) { + return database; + }, + }, discovery, });