Create separate roles and auto-create plugin databases

This would create a role per plugin (with a predictable pattern) before
creating a database owned by that role. This removes the need to
manually provision databases and users.

CHANGELOG.md

@@ -10,6 +10,9 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re
 
 ### Backend (example-backend, or backends created with @backstage/create-app)
 
+- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to:
+  - Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role)
+  - Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY <postgres user in app-config.yaml> TO <database owner role>`. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`.
 - The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562)
 - A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600)
 

packages/backend-common/src/database/connection.test.ts

@@ -27,9 +27,9 @@ describe('database connection', () => {
     ]);
 
   describe(createDatabaseClient, () => {
-    it('returns a postgres connection', () => {
+    it('returns a postgres connection', async () => {
       expect(
-        createDatabaseClient(
+        await createDatabaseClient(
           createConfig({
             client: 'pg',
             connection: {
@@ -43,9 +43,9 @@ describe('database connection', () => {
       ).toBeTruthy();
     });
 
-    it('returns an sqlite connection', () => {
+    it('returns an sqlite connection', async () => {
       expect(
-        createDatabaseClient(
+        await createDatabaseClient(
           createConfig({
             client: 'sqlite3',
             connection: ':memory:',
@@ -55,24 +55,25 @@ describe('database connection', () => {
     });
 
     it('tries to create a mysql connection as a passthrough', () => {
-      expect(() =>
-        createDatabaseClient(
-          createConfig({
-            client: 'mysql',
-            connection: {
-              host: '127.0.0.1',
-              user: 'foo',
-              password: 'bar',
-              database: 'dbname',
-            },
-          }),
-        ),
+      expect(
+        async () =>
+          await createDatabaseClient(
+            createConfig({
+              client: 'mysql',
+              connection: {
+                host: '127.0.0.1',
+                user: 'foo',
+                password: 'bar',
+                database: 'dbname',
+              },
+            }),
+          ),
       ).toThrowError(/Cannot find module 'mysql'/);
     });
 
-    it('accepts overrides', () => {
+    it('accepts overrides', async () => {
       expect(
-        createDatabaseClient(
+        await createDatabaseClient(
           createConfig({
             client: 'pg',
             connection: {
@@ -92,22 +93,24 @@ describe('database connection', () => {
     });
 
     it('throws an error without a client', () => {
-      expect(() =>
-        createDatabaseClient(
-          createConfig({
-            connection: '',
-          }),
-        ),
+      expect(
+        async () =>
+          await createDatabaseClient(
+            createConfig({
+              connection: '',
+            }),
+          ),
       ).toThrowError();
     });
 
     it('throws an error without a connection', () => {
-      expect(() =>
-        createDatabaseClient(
-          createConfig({
-            client: 'pg',
-          }),
-        ),
+      expect(
+        async () =>
+          await createDatabaseClient(
+            createConfig({
+              client: 'pg',
+            }),
+          ),
       ).toThrowError();
     });
   });

packages/backend-common/src/database/connection.ts

@@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string;
  * @param dbConfig The database config
  * @param overrides Additional options to merge with the config
  */
-export function createDatabaseClient(
+export async function createDatabaseClient(
   dbConfig: Config,
   overrides?: Partial<knex.Config>,
 ) {

packages/backend-common/src/database/postgres.test.ts

@@ -164,9 +164,9 @@ describe('postgres', () => {
   });
 
   describe(createPgDatabaseClient, () => {
-    it('creates a postgres knex instance', () => {
+    it('creates a postgres knex instance', async () => {
       expect(
-        createPgDatabaseClient(
+        await createPgDatabaseClient(
           createConfig({
             host: 'acme',
             user: 'foo',
@@ -177,13 +177,14 @@ describe('postgres', () => {
       ).toBeTruthy();
     });
 
-    it('attempts to read an ssl cert', () => {
-      expect(() =>
-        createPgDatabaseClient(
-          createConfig(
-            'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
+    it('attempts to read an ssl cert', async () => {
+      expect(
+        async () =>
+          await createPgDatabaseClient(
+            createConfig(
+              'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
+            ),
           ),
-        ),
       ).toThrowError(/no such file or directory/);
     });
   });

packages/backend-common/src/database/postgres.ts

@@ -15,26 +15,41 @@
  */
 
 import knex, { PgConnectionConfig } from 'knex';
-import { Config } from '@backstage/config';
+import { cloneDeep } from 'lodash';
+import { Config, JsonValue } from '@backstage/config';
 import { mergeDatabaseConfig } from './config';
 
 /**
- * Creates a knex postgres database connection
+ * Creates a knex Postgres database connection
  *
  * @param dbConfig The database config
  * @param overrides Additional options to merge with the config
  */
-export function createPgDatabaseClient(
+export async function createPgDatabaseClient(
   dbConfig: Config,
   overrides?: knex.Config,
 ) {
-  const knexConfig = buildPgDatabaseConfig(dbConfig, overrides);
+  const baseConfig = buildPgDatabaseConfig(dbConfig, overrides);
+  let knexConfig = baseConfig;
+
+  // Bootstrap the missing database.
+  if (!!baseConfig?.connection.database) {
+    const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig));
+    const admin = knex(knexAdminConfig);
+
+    await ensurePgDatabase(admin, baseConfig.connection.database);
+    knexConfig = buildPgPluginConfig(
+      cloneDeep(baseConfig),
+      baseConfig.connection.database,
+    );
+  }
+
   const database = knex(knexConfig);
   return database;
 }
 
 /**
- * Builds a knex postgres database connection
+ * Builds a knex Postgres database connection
  *
  * @param dbConfig The database config
  * @param overrides Additional options to merge with the config
@@ -44,7 +59,7 @@ export function buildPgDatabaseConfig(
   overrides?: knex.Config,
 ) {
   return mergeDatabaseConfig(
-    dbConfig.get(),
+    cloneDeep(dbConfig.get()),
     {
       connection: getPgConnectionConfig(dbConfig, !!overrides),
       useNullAsDefault: true,
@@ -54,10 +69,35 @@ export function buildPgDatabaseConfig(
 }
 
 /**
- * Gets the postgres connection config
+ * Builds a knex Postgres database connection for database creation
  *
  * @param dbConfig The database config
- * @param parseConnectionString Flag to explictly control connection string parsing
+ */
+function buildPgDatabaseAdminConfig(dbConfig: JsonValue) {
+  return mergeDatabaseConfig(dbConfig, {
+    connection: {
+      database: 'postgres',
+    },
+  });
+}
+
+/**
+ * Builds a knex Postgres database connection for plugin consumption
+ *
+ * @param dbConfig The database config
+ * @param database The database granted to the plugin
+ */
+function buildPgPluginConfig(dbConfig: JsonValue, database: string) {
+  return mergeDatabaseConfig(dbConfig, {
+    connection: roleCredentials(database),
+  });
+}
+
+/**
+ * Gets the Postgres connection config
+ *
+ * @param dbConfig The database config
+ * @param parseConnectionString Flag to explicitly control connection string parsing
  */
 export function getPgConnectionConfig(
   dbConfig: Config,
@@ -77,6 +117,41 @@ export function getPgConnectionConfig(
     : connection;
 }
 
+/**
+ * Creates the missing Postgres database if it does not exist
+ *
+ * @param admin The administrative database connection, defaulting to the `postgres` database
+ * @param database The name of the database to create
+ */
+async function ensurePgDatabase(admin: knex, database: string) {
+  const result = await admin
+    .from('pg_database')
+    .where('datname', database)
+    .count<Record<string, { count: string }>>();
+
+  if (parseInt(result[0].count, 10) > 0) {
+    return;
+  }
+
+  const owner = roleCredentials(database);
+  await admin.raw(`CREATE ROLE ?? WITH LOGIN PASSWORD '${owner.password}'`, [
+    owner.user,
+  ]);
+  await admin.raw(`CREATE DATABASE ?? OWNER ??`, [database, owner.user]);
+}
+
+/**
+ * Creates credentials to own the given database
+ *
+ * @param database The name of the database to create a role for
+ */
+function roleCredentials(database: string) {
+  return {
+    user: database,
+    password: database,
+  };
+}
+
 /**
  * Parses a connection string using pg-connection-string
  *

packages/backend/src/index.ts

@@ -49,9 +49,9 @@ import { PluginEnvironment } from './types';
 function makeCreateEnv(loadedConfigs: AppConfig[]) {
   const config = ConfigReader.fromConfigs(loadedConfigs);
 
-  return (plugin: string): PluginEnvironment => {
+  return async (plugin: string): Promise<PluginEnvironment> => {
     const logger = getRootLogger().child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
+    const database = await createDatabaseClient(
       config.getConfig('backend.database'),
       {
         connection: {
@@ -68,35 +68,41 @@ async function main() {
   const configs = await loadBackendConfig();
   const configReader = ConfigReader.fromConfigs(configs);
   const createEnv = makeCreateEnv(configs);
-  const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck'));
-  const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
-  const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));
-  const authEnv = useHotMemoize(module, () => createEnv('auth'));
-  const proxyEnv = useHotMemoize(module, () => createEnv('proxy'));
-  const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar'));
-  const sentryEnv = useHotMemoize(module, () => createEnv('sentry'));
-  const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs'));
-  const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes'));
-  const graphqlEnv = useHotMemoize(module, () => createEnv('graphql'));
-  const appEnv = useHotMemoize(module, () => createEnv('app'));
+  const healthcheckEnv = useHotMemoize(module, async () =>
+    createEnv('healthcheck'),
+  );
+  const catalogEnv = useHotMemoize(module, async () => createEnv('catalog'));
+  const scaffolderEnv = useHotMemoize(module, async () =>
+    createEnv('scaffolder'),
+  );
+  const authEnv = useHotMemoize(module, async () => createEnv('auth'));
+  const proxyEnv = useHotMemoize(module, async () => createEnv('proxy'));
+  const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar'));
+  const sentryEnv = useHotMemoize(module, async () => createEnv('sentry'));
+  const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs'));
+  const kubernetesEnv = useHotMemoize(module, async () =>
+    createEnv('kubernetes'),
+  );
+  const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql'));
+  const appEnv = useHotMemoize(module, async () => createEnv('app'));
 
   const apiRouter = Router();
-  apiRouter.use('/catalog', await catalog(catalogEnv));
-  apiRouter.use('/rollbar', await rollbar(rollbarEnv));
-  apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv));
-  apiRouter.use('/sentry', await sentry(sentryEnv));
-  apiRouter.use('/auth', await auth(authEnv));
-  apiRouter.use('/techdocs', await techdocs(techdocsEnv));
-  apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv));
-  apiRouter.use('/proxy', await proxy(proxyEnv));
-  apiRouter.use('/graphql', await graphql(graphqlEnv));
+  apiRouter.use('/catalog', await catalog(await catalogEnv));
+  apiRouter.use('/rollbar', await rollbar(await rollbarEnv));
+  apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv));
+  apiRouter.use('/sentry', await sentry(await sentryEnv));
+  apiRouter.use('/auth', await auth(await authEnv));
+  apiRouter.use('/techdocs', await techdocs(await techdocsEnv));
+  apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv));
+  apiRouter.use('/proxy', await proxy(await proxyEnv));
+  apiRouter.use('/graphql', await graphql(await graphqlEnv));
   apiRouter.use(notFoundHandler());
 
   const service = createServiceBuilder(module)
     .loadConfig(configReader)
-    .addRouter('', await healthcheck(healthcheckEnv))
+    .addRouter('', await healthcheck(await healthcheckEnv))
     .addRouter('/api', apiRouter)
-    .addRouter('', await app(appEnv));
+    .addRouter('', await app(await appEnv));
 
   await service.start().catch(err => {
     console.log(err);

packages/create-app/templates/default-app/packages/backend/src/index.ts

@@ -27,9 +27,9 @@ import { PluginEnvironment } from './types';
 function makeCreateEnv(loadedConfigs: AppConfig[]) {
   const config = ConfigReader.fromConfigs(loadedConfigs);
 
-  return (plugin: string): PluginEnvironment => {
+  return async (plugin: string): Promise<PluginEnvironment> => {
     const logger = getRootLogger().child({ type: 'plugin', plugin });
-    const database = createDatabaseClient(
+    const database = await createDatabaseClient(
       config.getConfig('backend.database'),
       {
         connection: {
@@ -47,18 +47,18 @@ async function main() {
   const configReader = ConfigReader.fromConfigs(configs);
   const createEnv = makeCreateEnv(configs);
 
-  const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
-  const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));
-  const authEnv = useHotMemoize(module, () => createEnv('auth'));
-  const proxyEnv = useHotMemoize(module, () => createEnv('proxy'));
-  const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs'));
+  const catalogEnv = useHotMemoize(module, async () => createEnv('catalog'));
+  const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder'));
+  const authEnv = useHotMemoize(module, async () => createEnv('auth'));
+  const proxyEnv = useHotMemoize(module, async () => createEnv('proxy'));
+  const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs'));
 
   const apiRouter = Router();
-  apiRouter.use('/catalog', await catalog(catalogEnv))
-  apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv))
-  apiRouter.use('/auth', await auth(authEnv))
-  apiRouter.use('/techdocs', await techdocs(techdocsEnv))
-  apiRouter.use('/proxy', await proxy(proxyEnv))
+  apiRouter.use('/catalog', await catalog(await catalogEnv))
+  apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv))
+  apiRouter.use('/auth', await auth(await authEnv))
+  apiRouter.use('/techdocs', await techdocs(await techdocsEnv))
+  apiRouter.use('/proxy', await proxy(await proxyEnv))
   apiRouter.use(notFoundHandler());
 
   const service = createServiceBuilder(module)