From b28c98a59742e5c9ec737c8c4fad3f425b1fd4f8 Mon Sep 17 00:00:00 2001 From: Joel Low Date: Wed, 30 Sep 2020 18:01:48 +0800 Subject: [PATCH 1/3] Create separate roles and auto-create plugin databases This would create a role per plugin (with a predictable pattern) before creating a database owned by that role. This removes the need to manually provision databases and users. --- CHANGELOG.md | 3 + .../src/database/connection.test.ts | 63 +++++++------ .../backend-common/src/database/connection.ts | 2 +- .../src/database/postgres.test.ts | 17 ++-- .../backend-common/src/database/postgres.ts | 91 +++++++++++++++++-- packages/backend/src/index.ts | 54 ++++++----- .../default-app/packages/backend/src/index.ts | 24 ++--- 7 files changed, 171 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2638cfe9..d3da193bba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re ### Backend (example-backend, or backends created with @backstage/create-app) +- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to: + - Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role) + - Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY TO `. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`. - The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562) - A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600) diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 2cc3a54e06..9584be882c 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -27,9 +27,9 @@ describe('database connection', () => { ]); describe(createDatabaseClient, () => { - it('returns a postgres connection', () => { + it('returns a postgres connection', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -43,9 +43,9 @@ describe('database connection', () => { ).toBeTruthy(); }); - it('returns an sqlite connection', () => { + it('returns an sqlite connection', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'sqlite3', connection: ':memory:', @@ -55,24 +55,25 @@ describe('database connection', () => { }); it('tries to create a mysql connection as a passthrough', () => { - expect(() => - createDatabaseClient( - createConfig({ - client: 'mysql', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + client: 'mysql', + connection: { + host: '127.0.0.1', + user: 'foo', + password: 'bar', + database: 'dbname', + }, + }), + ), ).toThrowError(/Cannot find module 'mysql'/); }); - it('accepts overrides', () => { + it('accepts overrides', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -92,22 +93,24 @@ describe('database connection', () => { }); it('throws an error without a client', () => { - expect(() => - createDatabaseClient( - createConfig({ - connection: '', - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + connection: '', + }), + ), ).toThrowError(); }); it('throws an error without a connection', () => { - expect(() => - createDatabaseClient( - createConfig({ - client: 'pg', - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + client: 'pg', + }), + ), ).toThrowError(); }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 009a85239f..2a1e1796d5 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string; * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export function createDatabaseClient( +export async function createDatabaseClient( dbConfig: Config, overrides?: Partial, ) { diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/postgres.test.ts index 82161dc53c..4d12551e53 100644 --- a/packages/backend-common/src/database/postgres.test.ts +++ b/packages/backend-common/src/database/postgres.test.ts @@ -164,9 +164,9 @@ describe('postgres', () => { }); describe(createPgDatabaseClient, () => { - it('creates a postgres knex instance', () => { + it('creates a postgres knex instance', async () => { expect( - createPgDatabaseClient( + await createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -177,13 +177,14 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', () => { - expect(() => - createPgDatabaseClient( - createConfig( - 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', + it('attempts to read an ssl cert', async () => { + expect( + async () => + await createPgDatabaseClient( + createConfig( + 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', + ), ), - ), ).toThrowError(/no such file or directory/); }); }); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 03a74156c7..bd8a9670cf 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -15,26 +15,41 @@ */ import knex, { PgConnectionConfig } from 'knex'; -import { Config } from '@backstage/config'; +import { cloneDeep } from 'lodash'; +import { Config, JsonValue } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; /** - * Creates a knex postgres database connection + * Creates a knex Postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export function createPgDatabaseClient( +export async function createPgDatabaseClient( dbConfig: Config, overrides?: knex.Config, ) { - const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); + const baseConfig = buildPgDatabaseConfig(dbConfig, overrides); + let knexConfig = baseConfig; + + // Bootstrap the missing database. + if (!!baseConfig?.connection.database) { + const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig)); + const admin = knex(knexAdminConfig); + + await ensurePgDatabase(admin, baseConfig.connection.database); + knexConfig = buildPgPluginConfig( + cloneDeep(baseConfig), + baseConfig.connection.database, + ); + } + const database = knex(knexConfig); return database; } /** - * Builds a knex postgres database connection + * Builds a knex Postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -44,7 +59,7 @@ export function buildPgDatabaseConfig( overrides?: knex.Config, ) { return mergeDatabaseConfig( - dbConfig.get(), + cloneDeep(dbConfig.get()), { connection: getPgConnectionConfig(dbConfig, !!overrides), useNullAsDefault: true, @@ -54,10 +69,35 @@ export function buildPgDatabaseConfig( } /** - * Gets the postgres connection config + * Builds a knex Postgres database connection for database creation * * @param dbConfig The database config - * @param parseConnectionString Flag to explictly control connection string parsing + */ +function buildPgDatabaseAdminConfig(dbConfig: JsonValue) { + return mergeDatabaseConfig(dbConfig, { + connection: { + database: 'postgres', + }, + }); +} + +/** + * Builds a knex Postgres database connection for plugin consumption + * + * @param dbConfig The database config + * @param database The database granted to the plugin + */ +function buildPgPluginConfig(dbConfig: JsonValue, database: string) { + return mergeDatabaseConfig(dbConfig, { + connection: roleCredentials(database), + }); +} + +/** + * Gets the Postgres connection config + * + * @param dbConfig The database config + * @param parseConnectionString Flag to explicitly control connection string parsing */ export function getPgConnectionConfig( dbConfig: Config, @@ -77,6 +117,41 @@ export function getPgConnectionConfig( : connection; } +/** + * Creates the missing Postgres database if it does not exist + * + * @param admin The administrative database connection, defaulting to the `postgres` database + * @param database The name of the database to create + */ +async function ensurePgDatabase(admin: knex, database: string) { + const result = await admin + .from('pg_database') + .where('datname', database) + .count>(); + + if (parseInt(result[0].count, 10) > 0) { + return; + } + + const owner = roleCredentials(database); + await admin.raw(`CREATE ROLE ?? WITH LOGIN PASSWORD '${owner.password}'`, [ + owner.user, + ]); + await admin.raw(`CREATE DATABASE ?? OWNER ??`, [database, owner.user]); +} + +/** + * Creates credentials to own the given database + * + * @param database The name of the database to create a role for + */ +function roleCredentials(database: string) { + return { + user: database, + password: database, + }; +} + /** * Parses a connection string using pg-connection-string * diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index dbd7842f8b..619de615c4 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -49,9 +49,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return (plugin: string): PluginEnvironment => { + return async (plugin: string): Promise => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = createDatabaseClient( + const database = await createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -68,35 +68,41 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); - const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); - const sentryEnv = useHotMemoize(module, () => createEnv('sentry')); - const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); - const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); - const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); - const appEnv = useHotMemoize(module, () => createEnv('app')); + const healthcheckEnv = useHotMemoize(module, async () => + createEnv('healthcheck'), + ); + const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, async () => + createEnv('scaffolder'), + ); + const authEnv = useHotMemoize(module, async () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); + const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar')); + const sentryEnv = useHotMemoize(module, async () => createEnv('sentry')); + const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); + const kubernetesEnv = useHotMemoize(module, async () => + createEnv('kubernetes'), + ); + const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql')); + const appEnv = useHotMemoize(module, async () => createEnv('app')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(catalogEnv)); - apiRouter.use('/rollbar', await rollbar(rollbarEnv)); - apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); - apiRouter.use('/sentry', await sentry(sentryEnv)); - apiRouter.use('/auth', await auth(authEnv)); - apiRouter.use('/techdocs', await techdocs(techdocsEnv)); - apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); - apiRouter.use('/proxy', await proxy(proxyEnv)); - apiRouter.use('/graphql', await graphql(graphqlEnv)); + apiRouter.use('/catalog', await catalog(await catalogEnv)); + apiRouter.use('/rollbar', await rollbar(await rollbarEnv)); + apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)); + apiRouter.use('/sentry', await sentry(await sentryEnv)); + apiRouter.use('/auth', await auth(await authEnv)); + apiRouter.use('/techdocs', await techdocs(await techdocsEnv)); + apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv)); + apiRouter.use('/proxy', await proxy(await proxyEnv)); + apiRouter.use('/graphql', await graphql(await graphqlEnv)); apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module) .loadConfig(configReader) - .addRouter('', await healthcheck(healthcheckEnv)) + .addRouter('', await healthcheck(await healthcheckEnv)) .addRouter('/api', apiRouter) - .addRouter('', await app(appEnv)); + .addRouter('', await app(await appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 07f1b1c99a..6c31d13781 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -27,9 +27,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return (plugin: string): PluginEnvironment => { + return async (plugin: string): Promise => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = createDatabaseClient( + const database = await createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -47,18 +47,18 @@ async function main() { const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); - const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, async () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); + const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(catalogEnv)) - apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)) - apiRouter.use('/auth', await auth(authEnv)) - apiRouter.use('/techdocs', await techdocs(techdocsEnv)) - apiRouter.use('/proxy', await proxy(proxyEnv)) + apiRouter.use('/catalog', await catalog(await catalogEnv)) + apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)) + apiRouter.use('/auth', await auth(await authEnv)) + apiRouter.use('/techdocs', await techdocs(await techdocsEnv)) + apiRouter.use('/proxy', await proxy(await proxyEnv)) apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module) From a9276c78c194acb485443631aba2b91a6704fe64 Mon Sep 17 00:00:00 2001 From: Joel Low Date: Thu, 1 Oct 2020 10:39:37 +0800 Subject: [PATCH 2/3] Remove role creation logic --- .../backend-common/src/database/postgres.ts | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index bd8a9670cf..6f7f706811 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -30,7 +30,7 @@ export async function createPgDatabaseClient( overrides?: knex.Config, ) { const baseConfig = buildPgDatabaseConfig(dbConfig, overrides); - let knexConfig = baseConfig; + const knexConfig = baseConfig; // Bootstrap the missing database. if (!!baseConfig?.connection.database) { @@ -38,10 +38,6 @@ export async function createPgDatabaseClient( const admin = knex(knexAdminConfig); await ensurePgDatabase(admin, baseConfig.connection.database); - knexConfig = buildPgPluginConfig( - cloneDeep(baseConfig), - baseConfig.connection.database, - ); } const database = knex(knexConfig); @@ -81,18 +77,6 @@ function buildPgDatabaseAdminConfig(dbConfig: JsonValue) { }); } -/** - * Builds a knex Postgres database connection for plugin consumption - * - * @param dbConfig The database config - * @param database The database granted to the plugin - */ -function buildPgPluginConfig(dbConfig: JsonValue, database: string) { - return mergeDatabaseConfig(dbConfig, { - connection: roleCredentials(database), - }); -} - /** * Gets the Postgres connection config * @@ -133,23 +117,7 @@ async function ensurePgDatabase(admin: knex, database: string) { return; } - const owner = roleCredentials(database); - await admin.raw(`CREATE ROLE ?? WITH LOGIN PASSWORD '${owner.password}'`, [ - owner.user, - ]); - await admin.raw(`CREATE DATABASE ?? OWNER ??`, [database, owner.user]); -} - -/** - * Creates credentials to own the given database - * - * @param database The name of the database to create a role for - */ -function roleCredentials(database: string) { - return { - user: database, - password: database, - }; + await admin.raw(`CREATE DATABASE ??`, [database]); } /** From 3f2ec3c080802c936bac15eea45228f766fa4b2f Mon Sep 17 00:00:00 2001 From: Joel Low Date: Thu, 1 Oct 2020 11:55:35 +0800 Subject: [PATCH 3/3] Let the Backend manage creating databases --- CHANGELOG.md | 3 - .../src/database/connection.test.ts | 63 ++++++------ .../backend-common/src/database/connection.ts | 20 +++- .../src/database/postgres.test.ts | 17 ++-- .../backend-common/src/database/postgres.ts | 95 +++++++++---------- packages/backend/src/index.ts | 59 ++++++------ .../default-app/packages/backend/src/index.ts | 30 +++--- 7 files changed, 148 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3da193bba..bf2638cfe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,6 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re ### Backend (example-backend, or backends created with @backstage/create-app) -- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to: - - Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role) - - Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY TO `. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`. - The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562) - A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600) diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 9584be882c..2cc3a54e06 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -27,9 +27,9 @@ describe('database connection', () => { ]); describe(createDatabaseClient, () => { - it('returns a postgres connection', async () => { + it('returns a postgres connection', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -43,9 +43,9 @@ describe('database connection', () => { ).toBeTruthy(); }); - it('returns an sqlite connection', async () => { + it('returns an sqlite connection', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'sqlite3', connection: ':memory:', @@ -55,25 +55,24 @@ describe('database connection', () => { }); it('tries to create a mysql connection as a passthrough', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - client: 'mysql', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + client: 'mysql', + connection: { + host: '127.0.0.1', + user: 'foo', + password: 'bar', + database: 'dbname', + }, + }), + ), ).toThrowError(/Cannot find module 'mysql'/); }); - it('accepts overrides', async () => { + it('accepts overrides', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -93,24 +92,22 @@ describe('database connection', () => { }); it('throws an error without a client', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - connection: '', - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + connection: '', + }), + ), ).toThrowError(); }); it('throws an error without a connection', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - client: 'pg', - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + client: 'pg', + }), + ), ).toThrowError(); }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 2a1e1796d5..4cd8606fc0 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -17,7 +17,7 @@ import knex from 'knex'; import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; -import { createPgDatabaseClient } from './postgres'; +import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres'; import { createSqliteDatabaseClient } from './sqlite3'; type DatabaseClient = 'pg' | 'sqlite3' | string; @@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string; * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export async function createDatabaseClient( +export function createDatabaseClient( dbConfig: Config, overrides?: Partial, ) { @@ -48,3 +48,19 @@ export async function createDatabaseClient( * @deprecated Use createDatabaseClient instead */ export const createDatabase = createDatabaseClient; + +/** + * Ensures that the given databases all exist, creating them if they do not. + */ +export async function ensureDatabaseExists( + dbConfig: Config, + ...databases: Array +) { + const client: DatabaseClient = dbConfig.getString('client'); + + if (client === 'pg') { + return ensurePgDatabaseExists(dbConfig, ...databases); + } + + return undefined; +} diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/postgres.test.ts index 4d12551e53..82161dc53c 100644 --- a/packages/backend-common/src/database/postgres.test.ts +++ b/packages/backend-common/src/database/postgres.test.ts @@ -164,9 +164,9 @@ describe('postgres', () => { }); describe(createPgDatabaseClient, () => { - it('creates a postgres knex instance', async () => { + it('creates a postgres knex instance', () => { expect( - await createPgDatabaseClient( + createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -177,14 +177,13 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', async () => { - expect( - async () => - await createPgDatabaseClient( - createConfig( - 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', - ), + it('attempts to read an ssl cert', () => { + expect(() => + createPgDatabaseClient( + createConfig( + 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', ), + ), ).toThrowError(/no such file or directory/); }); }); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 6f7f706811..82e971f081 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -15,37 +15,26 @@ */ import knex, { PgConnectionConfig } from 'knex'; -import { cloneDeep } from 'lodash'; -import { Config, JsonValue } from '@backstage/config'; +import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; /** - * Creates a knex Postgres database connection + * Creates a knex postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export async function createPgDatabaseClient( +export function createPgDatabaseClient( dbConfig: Config, overrides?: knex.Config, ) { - const baseConfig = buildPgDatabaseConfig(dbConfig, overrides); - const knexConfig = baseConfig; - - // Bootstrap the missing database. - if (!!baseConfig?.connection.database) { - const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig)); - const admin = knex(knexAdminConfig); - - await ensurePgDatabase(admin, baseConfig.connection.database); - } - + const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); const database = knex(knexConfig); return database; } /** - * Builds a knex Postgres database connection + * Builds a knex postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -55,7 +44,7 @@ export function buildPgDatabaseConfig( overrides?: knex.Config, ) { return mergeDatabaseConfig( - cloneDeep(dbConfig.get()), + dbConfig.get(), { connection: getPgConnectionConfig(dbConfig, !!overrides), useNullAsDefault: true, @@ -65,23 +54,10 @@ export function buildPgDatabaseConfig( } /** - * Builds a knex Postgres database connection for database creation + * Gets the postgres connection config * * @param dbConfig The database config - */ -function buildPgDatabaseAdminConfig(dbConfig: JsonValue) { - return mergeDatabaseConfig(dbConfig, { - connection: { - database: 'postgres', - }, - }); -} - -/** - * Gets the Postgres connection config - * - * @param dbConfig The database config - * @param parseConnectionString Flag to explicitly control connection string parsing + * @param parseConnectionString Flag to explictly control connection string parsing */ export function getPgConnectionConfig( dbConfig: Config, @@ -101,25 +77,6 @@ export function getPgConnectionConfig( : connection; } -/** - * Creates the missing Postgres database if it does not exist - * - * @param admin The administrative database connection, defaulting to the `postgres` database - * @param database The name of the database to create - */ -async function ensurePgDatabase(admin: knex, database: string) { - const result = await admin - .from('pg_database') - .where('datname', database) - .count>(); - - if (parseInt(result[0].count, 10) > 0) { - return; - } - - await admin.raw(`CREATE DATABASE ??`, [database]); -} - /** * Parses a connection string using pg-connection-string * @@ -138,3 +95,39 @@ function requirePgConnectionString() { throw new Error(`${message}\n${e.message}`); } } + +/** + * Creates the missing Postgres database if it does not exist + * + * @param dbConfig The database config + * @param databases The name of the databases to create + */ +export async function ensurePgDatabaseExists( + dbConfig: Config, + ...databases: Array +) { + const admin = createPgDatabaseClient(dbConfig, { + connection: { + database: 'postgres', + }, + }); + + try { + const ensureDatabase = async (database: string) => { + const result = await admin + .from('pg_database') + .where('datname', database) + .count>(); + + if (parseInt(result[0].count, 10) > 0) { + return; + } + + await admin.raw(`CREATE DATABASE ??`, [database]); + }; + + await Promise.all(databases.map(ensureDatabase)); + } finally { + await admin.destroy(); + } +} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 619de615c4..e86034dd29 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -24,6 +24,7 @@ import Router from 'express-promise-router'; import { + ensureDatabaseExists, createDatabaseClient, createServiceBuilder, loadBackendConfig, @@ -49,9 +50,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return async (plugin: string): Promise => { + return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = await createDatabaseClient( + const database = createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -68,41 +69,41 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const healthcheckEnv = useHotMemoize(module, async () => - createEnv('healthcheck'), + await ensureDatabaseExists( + configReader.getConfig('backend.database'), + 'backstage_plugin_catalog', + 'backstage_plugin_auth', ); - const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, async () => - createEnv('scaffolder'), - ); - const authEnv = useHotMemoize(module, async () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); - const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar')); - const sentryEnv = useHotMemoize(module, async () => createEnv('sentry')); - const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); - const kubernetesEnv = useHotMemoize(module, async () => - createEnv('kubernetes'), - ); - const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql')); - const appEnv = useHotMemoize(module, async () => createEnv('app')); + + const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); + const sentryEnv = useHotMemoize(module, () => createEnv('sentry')); + const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); + const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); + const appEnv = useHotMemoize(module, () => createEnv('app')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(await catalogEnv)); - apiRouter.use('/rollbar', await rollbar(await rollbarEnv)); - apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)); - apiRouter.use('/sentry', await sentry(await sentryEnv)); - apiRouter.use('/auth', await auth(await authEnv)); - apiRouter.use('/techdocs', await techdocs(await techdocsEnv)); - apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv)); - apiRouter.use('/proxy', await proxy(await proxyEnv)); - apiRouter.use('/graphql', await graphql(await graphqlEnv)); + apiRouter.use('/catalog', await catalog(catalogEnv)); + apiRouter.use('/rollbar', await rollbar(rollbarEnv)); + apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); + apiRouter.use('/sentry', await sentry(sentryEnv)); + apiRouter.use('/auth', await auth(authEnv)); + apiRouter.use('/techdocs', await techdocs(techdocsEnv)); + apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); + apiRouter.use('/proxy', await proxy(proxyEnv)); + apiRouter.use('/graphql', await graphql(graphqlEnv)); apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module) .loadConfig(configReader) - .addRouter('', await healthcheck(await healthcheckEnv)) + .addRouter('', await healthcheck(healthcheckEnv)) .addRouter('/api', apiRouter) - .addRouter('', await app(await appEnv)); + .addRouter('', await app(appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 6c31d13781..47ab2c5993 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -8,6 +8,7 @@ import Router from 'express-promise-router'; import { + ensureDatabaseExists, createDatabaseClient, createServiceBuilder, loadBackendConfig, @@ -27,9 +28,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return async (plugin: string): Promise => { + return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = await createDatabaseClient( + const database = createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -46,19 +47,24 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); + await ensureDatabaseExists( + configReader.getConfig('backend.database'), + 'backstage_plugin_catalog', + 'backstage_plugin_auth', + ); - const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, async () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); - const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(await catalogEnv)) - apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)) - apiRouter.use('/auth', await auth(await authEnv)) - apiRouter.use('/techdocs', await techdocs(await techdocsEnv)) - apiRouter.use('/proxy', await proxy(await proxyEnv)) + apiRouter.use('/catalog', await catalog(catalogEnv)) + apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)) + apiRouter.use('/auth', await auth(authEnv)) + apiRouter.use('/techdocs', await techdocs(techdocsEnv)) + apiRouter.use('/proxy', await proxy(proxyEnv)) apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module)