diff --git a/CHANGELOG.md b/CHANGELOG.md index 1860a4de6f..487b94a39d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re > Collect changes for the next release below +### Backend (example-backend, or backends created with @backstage/create-app) + +- A plugin database manager has been created, and plugins can now accept that interface as an argument during initialisation. Notably, the `auth` plugin has a [`createRouter` signature change](./plugins/auth-backend/src/service/router.ts). See [packages/backend/src/index.ts](./packages/backend/src/index.ts) on how to set it up. [#2697](https://github.com/spotify/backstage/pull/2697) + ## v0.1.1-alpha.24 ### Backend (example-backend, or backends created with @backstage/create-app) diff --git a/packages/backend-common/src/database/SingleConnection.test.ts b/packages/backend-common/src/database/SingleConnection.test.ts new file mode 100644 index 0000000000..6278e4ecad --- /dev/null +++ b/packages/backend-common/src/database/SingleConnection.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { createDatabaseClient } from './connection'; +import { SingleConnectionDatabaseManager } from './SingleConnection'; + +jest.mock('./connection'); + +describe('SingleConnectionDatabaseManager', () => { + const createConfig = (data: any) => + ConfigReader.fromConfigs([ + { + context: '', + data, + }, + ]); + + const defaultConfigOptions = { + backend: { + database: { + client: 'pg', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }; + const defaultConfig = () => createConfig(defaultConfigOptions); + + // This is similar to the ts-jest `mocked` helper. + const mocked = (f: Function) => f as jest.Mock; + + afterEach(() => jest.resetAllMocks()); + + describe('SingleConnectionDatabaseManager.fromConfig', () => { + it('accesses the backend.database key', () => { + const getConfig = jest.fn(); + const config = defaultConfig(); + config.getConfig = getConfig; + + SingleConnectionDatabaseManager.fromConfig(config); + + expect(getConfig.mock.calls[0][0]).toEqual('backend.database'); + }); + }); + + describe('SingleConnectionDatabaseManager.forPlugin', () => { + const manager = SingleConnectionDatabaseManager.fromConfig(defaultConfig()); + + it('connects to a database scoped to the plugin', async () => { + const pluginId = 'test1'; + await manager.forPlugin(pluginId).getClient(); + + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(1); + + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const callArgs = mockCalls[0]; + expect(callArgs[0].get()).toEqual(defaultConfigOptions.backend.database); + expect(callArgs[1].connection.database).toEqual( + `backstage_plugin_${pluginId}`, + ); + }); + + it('provides different plugins different databases', async () => { + const plugin1Id = 'test1'; + const plugin2Id = 'test2'; + await manager.forPlugin(plugin1Id).getClient(); + await manager.forPlugin(plugin2Id).getClient(); + + expect(mocked(createDatabaseClient)).toHaveBeenCalledTimes(2); + + const mockCalls = mocked(createDatabaseClient).mock.calls; + const plugin1CallArgs = mockCalls[0]; + const plugin2CallArgs = mockCalls[1]; + expect(plugin1CallArgs[1].connection.database).not.toEqual( + plugin2CallArgs[1].connection.database, + ); + }); + }); +}); diff --git a/packages/backend-common/src/database/SingleConnection.ts b/packages/backend-common/src/database/SingleConnection.ts new file mode 100644 index 0000000000..777ced54a0 --- /dev/null +++ b/packages/backend-common/src/database/SingleConnection.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Knex from 'knex'; +import { Config } from '@backstage/config'; +import { createDatabaseClient, ensureDatabaseExists } from './connection'; +import { PluginDatabaseManager } from './types'; + +/** + * Implements a Database Manager which will automatically create new databases + * for plugins when requested. All requested databases are created with the + * credentials provided; if the database already exists no attempt to create + * the database will be made. + */ +export class SingleConnectionDatabaseManager { + /** + * Creates a new SingleConnectionDatabaseManager instance by reading from the `backend` + * config section, specifically the `.database` key for discovering the management + * database configuration. + * + * @param config The loaded application configuration. + */ + static fromConfig(config: Config): SingleConnectionDatabaseManager { + return new SingleConnectionDatabaseManager( + config.getConfig('backend.database'), + ); + } + + private constructor(private readonly config: Config) {} + + /** + * Generates a PluginDatabaseManager for consumption by plugins. + * + * @param pluginId The plugin that the database manager should be created for. Plugin names should be unique. + */ + forPlugin(pluginId: string): PluginDatabaseManager { + const _this = this; + + return { + getClient(): Promise { + return _this.getDatabase(pluginId); + }, + }; + } + + private async getDatabase(pluginId: string): Promise { + const config = this.config; + const overrides = SingleConnectionDatabaseManager.getDatabaseOverrides( + pluginId, + ); + const overrideConfig = overrides.connection as Knex.ConnectionConfig; + await this.ensureDatabase(overrideConfig.database); + + return createDatabaseClient(config, overrides); + } + + private static getDatabaseOverrides(pluginId: string): Knex.Config { + return { + connection: { + database: `backstage_plugin_${pluginId}`, + }, + }; + } + + private async ensureDatabase(database: string) { + const config = this.config; + await ensureDatabaseExists(config, database); + } +} diff --git a/packages/backend-common/src/database/config.test.ts b/packages/backend-common/src/database/config.test.ts index ab29bc19ba..c9a4107686 100644 --- a/packages/backend-common/src/database/config.test.ts +++ b/packages/backend-common/src/database/config.test.ts @@ -18,6 +18,18 @@ import { mergeDatabaseConfig } from './config'; describe('config', () => { describe(mergeDatabaseConfig, () => { + it('does not mutate the input object', () => { + const input = { + original: 'key', + }; + const override = { + added: 'value', + }; + + mergeDatabaseConfig(input, override); + expect(input).not.toHaveProperty('added'); + }); + it('does not require overrides', () => { expect( mergeDatabaseConfig({ diff --git a/packages/backend-common/src/database/config.ts b/packages/backend-common/src/database/config.ts index 80abc06aa2..af32556c28 100644 --- a/packages/backend-common/src/database/config.ts +++ b/packages/backend-common/src/database/config.ts @@ -19,9 +19,9 @@ import { merge } from 'lodash'; /** * Merges database objects together * - * @param config The base config + * @param config The base config. The input is not modified * @param overrides Any additional overrides */ export function mergeDatabaseConfig(config: any, ...overrides: any[]) { - return merge(config, ...overrides); + return merge({}, config, ...overrides); } diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts index 38d3d6224b..c81153aa62 100644 --- a/packages/backend-common/src/database/index.ts +++ b/packages/backend-common/src/database/index.ts @@ -15,3 +15,5 @@ */ export * from './connection'; +export * from './types'; +export * from './SingleConnection'; diff --git a/packages/backend-common/src/database/types.ts b/packages/backend-common/src/database/types.ts new file mode 100644 index 0000000000..7eaa05f173 --- /dev/null +++ b/packages/backend-common/src/database/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; + +/** + * The PluginDatabaseManager manages access to databases that Plugins get. + */ +export interface PluginDatabaseManager { + /** + * getClient provides backend plugins database connections for itself. + * + * The purpose of this method is to allow plugins to get isolated data + * stores so that plugins are discouraged from database integration. + */ + getClient(): Promise; +} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 959af71427..18fd44f089 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -24,17 +24,16 @@ import Router from 'express-promise-router'; import { - ensureDatabaseExists, - createDatabaseClient, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, notFoundHandler, + SingleConnectionDatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; -import { ConfigReader, AppConfig } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; @@ -48,24 +47,18 @@ import graphql from './plugins/graphql'; import app from './plugins/app'; import { PluginEnvironment } from './types'; -function makeCreateEnv(loadedConfigs: AppConfig[]) { - const config = ConfigReader.fromConfigs(loadedConfigs); +function makeCreateEnv(config: ConfigReader) { const root = getRootLogger(); const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); + const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); + return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); - const database = createDatabaseClient( - config.getConfig('backend.database'), - { - connection: { - database: `backstage_plugin_${plugin}`, - }, - }, - ); + const database = databaseManager.forPlugin(plugin); return { logger, database, config, reader, discovery }; }; } @@ -73,12 +66,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); - const createEnv = makeCreateEnv(configs); - await ensureDatabaseExists( - configReader.getConfig('backend.database'), - 'backstage_plugin_catalog', - 'backstage_plugin_auth', - ); + const createEnv = makeCreateEnv(configReader); const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index c1d390f8cc..b8e99e6398 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -34,7 +34,9 @@ export default async function createPlugin({ }: PluginEnvironment) { const locationReader = new LocationReaders({ logger, reader, config }); - const db = await DatabaseManager.createDatabase(database, { logger }); + const db = await DatabaseManager.createDatabase(await database.getClient(), { + logger, + }); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); const higherOrderOperation = new HigherOrderOperations( diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 9257fcc9cf..f63b9dd780 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -14,14 +14,17 @@ * limitations under the License. */ -import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; +import { + PluginDatabaseManager, + PluginEndpointDiscovery, + UrlReader, +} from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; - database: Knex; + database: PluginDatabaseManager; config: Config; reader: UrlReader; discovery: PluginEndpointDiscovery; diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 684db8aab8..d58ebe4e42 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -8,17 +8,16 @@ import Router from 'express-promise-router'; import { - ensureDatabaseExists, - createDatabaseClient, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, notFoundHandler, + SingleConnectionDatabaseManager, SingleHostDiscovery, UrlReaders, } from '@backstage/backend-common'; -import { ConfigReader, AppConfig } from '@backstage/config'; +import { ConfigReader } from '@backstage/config'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import scaffolder from './plugins/scaffolder'; @@ -26,24 +25,18 @@ import proxy from './plugins/proxy'; import techdocs from './plugins/techdocs'; import { PluginEnvironment } from './types'; -function makeCreateEnv(loadedConfigs: AppConfig[]) { - const config = ConfigReader.fromConfigs(loadedConfigs); +function makeCreateEnv(config: ConfigReader) { const root = getRootLogger(); const reader = UrlReaders.default({ logger: root, config }); const discovery = SingleHostDiscovery.fromConfig(config); root.info(`Created UrlReader ${reader}`); + const databaseManager = SingleConnectionDatabaseManager.fromConfig(config); + return (plugin: string): PluginEnvironment => { const logger = root.child({ type: 'plugin', plugin }); - const database = createDatabaseClient( - config.getConfig('backend.database'), - { - connection: { - database: `backstage_plugin_${plugin}`, - }, - }, - ); + const database = databaseManager.forPlugin(plugin); return { logger, database, config, reader, discovery }; }; } @@ -51,12 +44,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); - const createEnv = makeCreateEnv(configs); - await ensureDatabaseExists( - configReader.getConfig('backend.database'), - 'backstage_plugin_catalog', - 'backstage_plugin_auth', - ); + const createEnv = makeCreateEnv(configReader); const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts index 345ac4c23d..aee38a5fa0 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts @@ -18,7 +18,9 @@ export default async function createPlugin({ }: PluginEnvironment) { const locationReader = new LocationReaders({ logger, reader, config }); - const db = await DatabaseManager.createDatabase(database, { logger }); + const db = await DatabaseManager.createDatabase(await database.getClient(), + { logger }, + ); const entitiesCatalog = new DatabaseEntitiesCatalog(db); const locationsCatalog = new DatabaseLocationsCatalog(db); const higherOrderOperation = new HigherOrderOperations( diff --git a/packages/create-app/templates/default-app/packages/backend/src/types.ts b/packages/create-app/templates/default-app/packages/backend/src/types.ts index aa003f63a5..757a0e5acf 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/types.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/types.ts @@ -1,11 +1,14 @@ -import Knex from 'knex'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; +import { + PluginDatabaseManager, + PluginEndpointDiscovery, + UrlReader, +} from '@backstage/backend-common'; export type PluginEnvironment = { logger: Logger; - database: Knex; + database: PluginDatabaseManager; config: Config; reader: UrlReader discovery: PluginEndpointDiscovery; diff --git a/packages/e2e-test/src/e2e-test.ts b/packages/e2e-test/src/e2e-test.ts index 6fbdbf3c88..fe3a9f6bb4 100644 --- a/packages/e2e-test/src/e2e-test.ts +++ b/packages/e2e-test/src/e2e-test.ts @@ -328,8 +328,8 @@ async function testAppServe(pluginName: string, appDir: string) { } } -/** Creates PG databases (drops if exists before) */ -async function createDB(database: string) { +/** Drops PG databases */ +async function dropDB(database: string) { const config = { host: process.env.POSTGRES_HOST, port: process.env.POSTGRES_PORT, @@ -342,7 +342,6 @@ async function createDB(database: string) { } catch (_) { /* do nothing*/ } - return pgtools.createdb(config, database); } /** @@ -350,7 +349,7 @@ async function createDB(database: string) { */ async function testBackendStart(appDir: string, isPostgres: boolean) { if (isPostgres) { - print('Creating DBs'); + print('Dropping old DBs'); await Promise.all( [ 'catalog', @@ -359,7 +358,7 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { 'identity', 'proxy', 'techdocs', - ].map(name => createDB(`backstage_plugin_${name}`)), + ].map(name => dropDB(`backstage_plugin_${name}`)), ); print('Created DBs'); } diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index bcd1fc1794..6bb100de6b 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -17,7 +17,6 @@ import express from 'express'; import Router from 'express-promise-router'; import cookieParser from 'cookie-parser'; -import Knex from 'knex'; import { Logger } from 'winston'; import { createAuthProvider } from '../providers'; import { Config } from '@backstage/config'; @@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity'; import { NotFoundError, PluginEndpointDiscovery, + PluginDatabaseManager, } from '@backstage/backend-common'; export interface RouterOptions { logger: Logger; - database: Knex; + database: PluginDatabaseManager; config: Config; discovery: PluginEndpointDiscovery; } @@ -47,7 +47,9 @@ export async function createRouter({ const keyDurationSeconds = 3600; - const keyStore = await DatabaseKeyStore.create({ database }); + const keyStore = await DatabaseKeyStore.create({ + database: await database.getClient(), + }); const tokenIssuer = new TokenFactory({ issuer: authUrl, keyStore, diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts index 71771340a5..d00e7d523d 100644 --- a/plugins/auth-backend/src/service/standaloneServer.ts +++ b/plugins/auth-backend/src/service/standaloneServer.ts @@ -53,7 +53,11 @@ export async function startStandaloneServer( const router = await createRouter({ logger, config, - database, + database: { + async getClient() { + return database; + }, + }, discovery, });