diff --git a/CHANGELOG.md b/CHANGELOG.md index d3da193bba..bf2638cfe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,6 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re ### Backend (example-backend, or backends created with @backstage/create-app) -- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to: - - Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role) - - Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY TO `. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`. - The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562) - A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600) diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 9584be882c..2cc3a54e06 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -27,9 +27,9 @@ describe('database connection', () => { ]); describe(createDatabaseClient, () => { - it('returns a postgres connection', async () => { + it('returns a postgres connection', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -43,9 +43,9 @@ describe('database connection', () => { ).toBeTruthy(); }); - it('returns an sqlite connection', async () => { + it('returns an sqlite connection', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'sqlite3', connection: ':memory:', @@ -55,25 +55,24 @@ describe('database connection', () => { }); it('tries to create a mysql connection as a passthrough', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - client: 'mysql', - connection: { - host: '127.0.0.1', - user: 'foo', - password: 'bar', - database: 'dbname', - }, - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + client: 'mysql', + connection: { + host: '127.0.0.1', + user: 'foo', + password: 'bar', + database: 'dbname', + }, + }), + ), ).toThrowError(/Cannot find module 'mysql'/); }); - it('accepts overrides', async () => { + it('accepts overrides', () => { expect( - await createDatabaseClient( + createDatabaseClient( createConfig({ client: 'pg', connection: { @@ -93,24 +92,22 @@ describe('database connection', () => { }); it('throws an error without a client', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - connection: '', - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + connection: '', + }), + ), ).toThrowError(); }); it('throws an error without a connection', () => { - expect( - async () => - await createDatabaseClient( - createConfig({ - client: 'pg', - }), - ), + expect(() => + createDatabaseClient( + createConfig({ + client: 'pg', + }), + ), ).toThrowError(); }); }); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 2a1e1796d5..4cd8606fc0 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -17,7 +17,7 @@ import knex from 'knex'; import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; -import { createPgDatabaseClient } from './postgres'; +import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres'; import { createSqliteDatabaseClient } from './sqlite3'; type DatabaseClient = 'pg' | 'sqlite3' | string; @@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string; * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export async function createDatabaseClient( +export function createDatabaseClient( dbConfig: Config, overrides?: Partial, ) { @@ -48,3 +48,19 @@ export async function createDatabaseClient( * @deprecated Use createDatabaseClient instead */ export const createDatabase = createDatabaseClient; + +/** + * Ensures that the given databases all exist, creating them if they do not. + */ +export async function ensureDatabaseExists( + dbConfig: Config, + ...databases: Array +) { + const client: DatabaseClient = dbConfig.getString('client'); + + if (client === 'pg') { + return ensurePgDatabaseExists(dbConfig, ...databases); + } + + return undefined; +} diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/postgres.test.ts index 4d12551e53..82161dc53c 100644 --- a/packages/backend-common/src/database/postgres.test.ts +++ b/packages/backend-common/src/database/postgres.test.ts @@ -164,9 +164,9 @@ describe('postgres', () => { }); describe(createPgDatabaseClient, () => { - it('creates a postgres knex instance', async () => { + it('creates a postgres knex instance', () => { expect( - await createPgDatabaseClient( + createPgDatabaseClient( createConfig({ host: 'acme', user: 'foo', @@ -177,14 +177,13 @@ describe('postgres', () => { ).toBeTruthy(); }); - it('attempts to read an ssl cert', async () => { - expect( - async () => - await createPgDatabaseClient( - createConfig( - 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', - ), + it('attempts to read an ssl cert', () => { + expect(() => + createPgDatabaseClient( + createConfig( + 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', ), + ), ).toThrowError(/no such file or directory/); }); }); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts index 6f7f706811..82e971f081 100644 --- a/packages/backend-common/src/database/postgres.ts +++ b/packages/backend-common/src/database/postgres.ts @@ -15,37 +15,26 @@ */ import knex, { PgConnectionConfig } from 'knex'; -import { cloneDeep } from 'lodash'; -import { Config, JsonValue } from '@backstage/config'; +import { Config } from '@backstage/config'; import { mergeDatabaseConfig } from './config'; /** - * Creates a knex Postgres database connection + * Creates a knex postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config */ -export async function createPgDatabaseClient( +export function createPgDatabaseClient( dbConfig: Config, overrides?: knex.Config, ) { - const baseConfig = buildPgDatabaseConfig(dbConfig, overrides); - const knexConfig = baseConfig; - - // Bootstrap the missing database. - if (!!baseConfig?.connection.database) { - const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig)); - const admin = knex(knexAdminConfig); - - await ensurePgDatabase(admin, baseConfig.connection.database); - } - + const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); const database = knex(knexConfig); return database; } /** - * Builds a knex Postgres database connection + * Builds a knex postgres database connection * * @param dbConfig The database config * @param overrides Additional options to merge with the config @@ -55,7 +44,7 @@ export function buildPgDatabaseConfig( overrides?: knex.Config, ) { return mergeDatabaseConfig( - cloneDeep(dbConfig.get()), + dbConfig.get(), { connection: getPgConnectionConfig(dbConfig, !!overrides), useNullAsDefault: true, @@ -65,23 +54,10 @@ export function buildPgDatabaseConfig( } /** - * Builds a knex Postgres database connection for database creation + * Gets the postgres connection config * * @param dbConfig The database config - */ -function buildPgDatabaseAdminConfig(dbConfig: JsonValue) { - return mergeDatabaseConfig(dbConfig, { - connection: { - database: 'postgres', - }, - }); -} - -/** - * Gets the Postgres connection config - * - * @param dbConfig The database config - * @param parseConnectionString Flag to explicitly control connection string parsing + * @param parseConnectionString Flag to explictly control connection string parsing */ export function getPgConnectionConfig( dbConfig: Config, @@ -101,25 +77,6 @@ export function getPgConnectionConfig( : connection; } -/** - * Creates the missing Postgres database if it does not exist - * - * @param admin The administrative database connection, defaulting to the `postgres` database - * @param database The name of the database to create - */ -async function ensurePgDatabase(admin: knex, database: string) { - const result = await admin - .from('pg_database') - .where('datname', database) - .count>(); - - if (parseInt(result[0].count, 10) > 0) { - return; - } - - await admin.raw(`CREATE DATABASE ??`, [database]); -} - /** * Parses a connection string using pg-connection-string * @@ -138,3 +95,39 @@ function requirePgConnectionString() { throw new Error(`${message}\n${e.message}`); } } + +/** + * Creates the missing Postgres database if it does not exist + * + * @param dbConfig The database config + * @param databases The name of the databases to create + */ +export async function ensurePgDatabaseExists( + dbConfig: Config, + ...databases: Array +) { + const admin = createPgDatabaseClient(dbConfig, { + connection: { + database: 'postgres', + }, + }); + + try { + const ensureDatabase = async (database: string) => { + const result = await admin + .from('pg_database') + .where('datname', database) + .count>(); + + if (parseInt(result[0].count, 10) > 0) { + return; + } + + await admin.raw(`CREATE DATABASE ??`, [database]); + }; + + await Promise.all(databases.map(ensureDatabase)); + } finally { + await admin.destroy(); + } +} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 619de615c4..e86034dd29 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -24,6 +24,7 @@ import Router from 'express-promise-router'; import { + ensureDatabaseExists, createDatabaseClient, createServiceBuilder, loadBackendConfig, @@ -49,9 +50,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return async (plugin: string): Promise => { + return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = await createDatabaseClient( + const database = createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -68,41 +69,41 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); - const healthcheckEnv = useHotMemoize(module, async () => - createEnv('healthcheck'), + await ensureDatabaseExists( + configReader.getConfig('backend.database'), + 'backstage_plugin_catalog', + 'backstage_plugin_auth', ); - const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, async () => - createEnv('scaffolder'), - ); - const authEnv = useHotMemoize(module, async () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); - const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar')); - const sentryEnv = useHotMemoize(module, async () => createEnv('sentry')); - const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); - const kubernetesEnv = useHotMemoize(module, async () => - createEnv('kubernetes'), - ); - const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql')); - const appEnv = useHotMemoize(module, async () => createEnv('app')); + + const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck')); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); + const sentryEnv = useHotMemoize(module, () => createEnv('sentry')); + const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); + const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); + const appEnv = useHotMemoize(module, () => createEnv('app')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(await catalogEnv)); - apiRouter.use('/rollbar', await rollbar(await rollbarEnv)); - apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)); - apiRouter.use('/sentry', await sentry(await sentryEnv)); - apiRouter.use('/auth', await auth(await authEnv)); - apiRouter.use('/techdocs', await techdocs(await techdocsEnv)); - apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv)); - apiRouter.use('/proxy', await proxy(await proxyEnv)); - apiRouter.use('/graphql', await graphql(await graphqlEnv)); + apiRouter.use('/catalog', await catalog(catalogEnv)); + apiRouter.use('/rollbar', await rollbar(rollbarEnv)); + apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); + apiRouter.use('/sentry', await sentry(sentryEnv)); + apiRouter.use('/auth', await auth(authEnv)); + apiRouter.use('/techdocs', await techdocs(techdocsEnv)); + apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); + apiRouter.use('/proxy', await proxy(proxyEnv)); + apiRouter.use('/graphql', await graphql(graphqlEnv)); apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module) .loadConfig(configReader) - .addRouter('', await healthcheck(await healthcheckEnv)) + .addRouter('', await healthcheck(healthcheckEnv)) .addRouter('/api', apiRouter) - .addRouter('', await app(await appEnv)); + .addRouter('', await app(appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 6c31d13781..47ab2c5993 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -8,6 +8,7 @@ import Router from 'express-promise-router'; import { + ensureDatabaseExists, createDatabaseClient, createServiceBuilder, loadBackendConfig, @@ -27,9 +28,9 @@ import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { const config = ConfigReader.fromConfigs(loadedConfigs); - return async (plugin: string): Promise => { + return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - const database = await createDatabaseClient( + const database = createDatabaseClient( config.getConfig('backend.database'), { connection: { @@ -46,19 +47,24 @@ async function main() { const configs = await loadBackendConfig(); const configReader = ConfigReader.fromConfigs(configs); const createEnv = makeCreateEnv(configs); + await ensureDatabaseExists( + configReader.getConfig('backend.database'), + 'backstage_plugin_catalog', + 'backstage_plugin_auth', + ); - const catalogEnv = useHotMemoize(module, async () => createEnv('catalog')); - const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder')); - const authEnv = useHotMemoize(module, async () => createEnv('auth')); - const proxyEnv = useHotMemoize(module, async () => createEnv('proxy')); - const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs')); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); + const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); + const authEnv = useHotMemoize(module, () => createEnv('auth')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const apiRouter = Router(); - apiRouter.use('/catalog', await catalog(await catalogEnv)) - apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv)) - apiRouter.use('/auth', await auth(await authEnv)) - apiRouter.use('/techdocs', await techdocs(await techdocsEnv)) - apiRouter.use('/proxy', await proxy(await proxyEnv)) + apiRouter.use('/catalog', await catalog(catalogEnv)) + apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)) + apiRouter.use('/auth', await auth(authEnv)) + apiRouter.use('/techdocs', await techdocs(techdocsEnv)) + apiRouter.use('/proxy', await proxy(proxyEnv)) apiRouter.use(notFoundHandler()); const service = createServiceBuilder(module)