diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2638cfe9..d3da193bba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re ### Backend (example-backend, or backends created with @backstage/create-app) +- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to: + - Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role) + - Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY TO `. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`. - The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562) - A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600) diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 2cc3a54e06..9584be882c 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -27,9 +27,9 @@ describe('database connection', () => { ]); describe(createDatabaseClient, () => { - it('returns a postgres connection', () => { + it('returns a postgres connection', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -43,9 +43,9 @@ describe('database connection', () => { ).toBeTruthy(); }); - it('returns an sqlite connection', () => { + it('returns an sqlite connection', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'sqlite3', connection: ':memory:', @@ -55,24 +55,25 @@ describe('database connection', () => { }); it('tries to create a mysql connection as a passthrough', () => { - expect(() => - createDatabaseClient( - createConfig({ - client: 'mysql', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + client: 'mysql', + connection: { + host: '127.0.0.1', + user: 'foo', + password: 'bar', + database: 'dbname', + }, + }), + ), ).toThrowError(/Cannot find module 'mysql'/); }); - it('accepts overrides', () => { + it('accepts overrides', async () => { expect( - createDatabaseClient( + await createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -92,22 +93,24 @@ describe('database connection', () => { }); it('throws an error without a client', () => { - expect(() => - createDatabaseClient( - createConfig({ - connection: '', - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + connection: '', + }), + ), ).toThrowError(); }); it('throws an error without a connection', () => { - expect(() => - createDatabaseClient( - createConfig({ - client: 'pg', - }), - ), + expect( + async () => + await createDatabaseClient( + createConfig({ + client: 'pg', + }), + ), ).toThrowError(); }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 009a85239f..2a1e1796d5 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string; * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export function createDatabaseClient( +export async function createDatabaseClient( dbConfig: Config, overrides?: Partial, ) { diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/postgres.test.ts index 82161dc53c..4d12551e53 100644 --- a/packages/backend-common/src/database/postgres.test.ts +++ b/packages/backend-common/src/database/postgres.test.ts @@ -164,9 +164,9 @@ describe('postgres', () => { }); describe(createPgDatabaseClient, () => { - it('creates a postgres knex instance', () => { + it('creates a postgres knex instance', async () => { expect( - createPgDatabaseClient( + await createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -177,13 +177,14 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', () => { - expect(() => - createPgDatabaseClient( - createConfig( - 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', + it('attempts to read an ssl cert', async () => { + expect( + async () => + await createPgDatabaseClient( + createConfig( + 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', + ), ), - ), ).toThrowError(/no such file or directory/); }); }); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 03a74156c7..bd8a9670cf 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -15,26 +15,41 @@ */ import knex, { PgConnectionConfig } from 'knex'; -import { Config } from '@backstage/config'; +import { cloneDeep } from 'lodash'; +import { Config, JsonValue } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; /** - * Creates a knex postgres database connection + * Creates a knex Postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export function createPgDatabaseClient( +export async function createPgDatabaseClient( dbConfig: Config, overrides?: knex.Config, ) { - const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); + const baseConfig = buildPgDatabaseConfig(dbConfig, overrides); + let knexConfig = baseConfig; + + // Bootstrap the missing database. + if (!!baseConfig?.connection.database) { + const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig)); + const admin = knex(knexAdminConfig); + + await ensurePgDatabase(admin, baseConfig.connection.database); + knexConfig = buildPgPluginConfig( + cloneDeep(baseConfig), + baseConfig.connection.database, + ); + } + const database = knex(knexConfig); return database; } /** - * Builds a knex postgres database connection + * Builds a knex Postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -44,7 +59,7 @@ export function buildPgDatabaseConfig( overrides?: knex.Config, ) { return mergeDatabaseConfig( - dbConfig.get(), + cloneDeep(dbConfig.get()), { connection: getPgConnectionConfig(dbConfig, !!overrides), useNullAsDefault: true, @@ -54,10 +69,35 @@ export function buildPgDatabaseConfig( } /** - * Gets the postgres connection config + * Builds a knex Postgres database connection for database creation * * @param dbConfig The database config - * @param parseConnectionString Flag to explictly control connection string parsing + */ +function buildPgDatabaseAdminConfig(dbConfig: JsonValue) { + return mergeDatabaseConfig(dbConfig, { + connection: { + database: 'postgres', + }, + }); +} + +/** + * Builds a knex Postgres database connection for plugin consumption + * + * @param dbConfig The database config + * @param database The database granted to the plugin + */ +function buildPgPluginConfig(dbConfig: JsonValue, database: string) { + return mergeDatabaseConfig(dbConfig, { + connection: roleCredentials(database), + }); +} + +/** + * Gets the Postgres connection config + * + * @param dbConfig The database config + * @param parseConnectionString Flag to explicitly control connection string parsing */ export function getPgConnectionConfig( dbConfig: Config, @@ -77,6 +117,41 @@ export function getPgConnectionConfig( : connection; } +/** + * Creates the missing Postgres database if it does not exist + * + * @param admin The administrative database connection, defaulting to the `postgres` database + * @param database The name of the database to create + */ +async function ensurePgDatabase(admin: knex, database: string) { + const result = await admin + .from('pg_database') + .where('datname', database) + .count>(); + + if (parseInt(result[0].count, 10) > 0) { + return; + } + + const owner = roleCredentials(database); + await admin.raw(`CREATE ROLE ?? WITH LOGIN PASSWORD '${owner.password}'`, [ + owner.user, + ]); + await admin.raw(`CREATE DATABASE ?? OWNER ??`, [database, owner.user]); +} + +/** + * Creates credentials to own the given database + * + * @param database The name of the database to create a role for + */ +function roleCredentials(database: string) { + return { + user: database, + password: database, + }; +} + /** * Parses a connection string using pg-connection-string * diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index dbd7842f8b..619de615c4 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -49,9 +49,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return (plugin: string): PluginEnvironment => { + return async (plugin: string): Promise => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = createDatabaseClient( + const database = await createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -68,35 +68,41 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); - const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); - const sentryEnv = useHotMemoize(module, () => createEnv('sentry')); - const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); - const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); - const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); - const appEnv = useHotMemoize(module, () => createEnv('app')); + const healthcheckEnv = useHotMemoize(module, async () => + createEnv('healthcheck'), + ); + const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, async () => + createEnv('scaffolder'), + ); + const authEnv = useHotMemoize(module, async () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); + const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar')); + const sentryEnv = useHotMemoize(module, async () => createEnv('sentry')); + const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); + const kubernetesEnv = useHotMemoize(module, async () => + createEnv('kubernetes'), + ); + const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql')); + const appEnv = useHotMemoize(module, async () => createEnv('app')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(catalogEnv)); - apiRouter.use('/rollbar', await rollbar(rollbarEnv)); - apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); - apiRouter.use('/sentry', await sentry(sentryEnv)); - apiRouter.use('/auth', await auth(authEnv)); - apiRouter.use('/techdocs', await techdocs(techdocsEnv)); - apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); - apiRouter.use('/proxy', await proxy(proxyEnv)); - apiRouter.use('/graphql', await graphql(graphqlEnv)); + apiRouter.use('/catalog', await catalog(await catalogEnv)); + apiRouter.use('/rollbar', await rollbar(await rollbarEnv)); + apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)); + apiRouter.use('/sentry', await sentry(await sentryEnv)); + apiRouter.use('/auth', await auth(await authEnv)); + apiRouter.use('/techdocs', await techdocs(await techdocsEnv)); + apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv)); + apiRouter.use('/proxy', await proxy(await proxyEnv)); + apiRouter.use('/graphql', await graphql(await graphqlEnv)); apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module) .loadConfig(configReader) - .addRouter('', await healthcheck(healthcheckEnv)) + .addRouter('', await healthcheck(await healthcheckEnv)) .addRouter('/api', apiRouter) - .addRouter('', await app(appEnv)); + .addRouter('', await app(await appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 07f1b1c99a..6c31d13781 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -27,9 +27,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return (plugin: string): PluginEnvironment => { + return async (plugin: string): Promise => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = createDatabaseClient( + const database = await createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -47,18 +47,18 @@ async function main() { const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); - const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, async () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); + const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(catalogEnv)) - apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)) - apiRouter.use('/auth', await auth(authEnv)) - apiRouter.use('/techdocs', await techdocs(techdocsEnv)) - apiRouter.use('/proxy', await proxy(proxyEnv)) + apiRouter.use('/catalog', await catalog(await catalogEnv)) + apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)) + apiRouter.use('/auth', await auth(await authEnv)) + apiRouter.use('/techdocs', await techdocs(await techdocsEnv)) + apiRouter.use('/proxy', await proxy(await proxyEnv)) apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module)