paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

Let the Backend manage creating databases

Browse Source
This commit is contained in:
Joel Low
2020-10-01 11:55:35 +08:00
parent a9276c78c1
commit 3f2ec3c080
7 changed files with 148 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
-3
View File
@@ -10,9 +10,6 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re
### Backend (example-backend, or backends created with @backstage/create-app)
- Backends configured with a Postgres database would require manual migration. Previous versions of Backstage would rely on a user-provided database, using the configured user in `app-config.yaml`. Backstage will now manage the databases and roles in the provided database instance; existing users would need to:
- Create a role for each Backstage database (e.g. create the `backstage_plugin_catalog` role)
- Change the ownership of each Backstage database to the corresponding role. In each database, run `REASSIGN OWNED BY <postgres user in app-config.yaml> TO <database owner role>`. For example, in the `backstage_plugin_catalog` database, run `REASSIGN OWNED BY postgres TO backstage_plugin_catalog`.
- The default mount point for backend plugins have been changed to `/api`. These changes are done in the backend package itself, so it is recommended that you sync up existing backend packages with this new pattern. [#2562](https://github.com/spotify/backstage/pull/2562)
- A service discovery mechanism for backend plugins has been added, and is now a requirement for several backend plugins. See [packages/backend/src/index.ts](./packages/backend/src/index.ts) for how to set it up using `SingleHostDiscovery` from `@backstage/backend-common`. Note that the default base path for plugins is set to `/api` to that change, but it can be set to use the old behavior via the `basePath` option. [#2600](https://github.com/spotify/backstage/pull/2600)
packages/backend-common/src/database/connection.test.ts
+30 -33
View File
@@ -27,9 +27,9 @@ describe('database connection', () => {
]);
describe(createDatabaseClient, () => {
it('returns a postgres connection', async () => {
it('returns a postgres connection', () => {
expect(
await createDatabaseClient(
createDatabaseClient(
createConfig({
client: 'pg',
connection: {
@@ -43,9 +43,9 @@ describe('database connection', () => {
).toBeTruthy();
});
it('returns an sqlite connection', async () => {
it('returns an sqlite connection', () => {
expect(
await createDatabaseClient(
createDatabaseClient(
createConfig({
client: 'sqlite3',
connection: ':memory:',
@@ -55,25 +55,24 @@ describe('database connection', () => {
});
it('tries to create a mysql connection as a passthrough', () => {
expect(
async () =>
await createDatabaseClient(
createConfig({
client: 'mysql',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
}),
),
expect(() =>
createDatabaseClient(
createConfig({
client: 'mysql',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
}),
),
).toThrowError(/Cannot find module 'mysql'/);
});
it('accepts overrides', async () => {
it('accepts overrides', () => {
expect(
await createDatabaseClient(
createDatabaseClient(
createConfig({
client: 'pg',
connection: {
@@ -93,24 +92,22 @@ describe('database connection', () => {
});
it('throws an error without a client', () => {
expect(
async () =>
await createDatabaseClient(
createConfig({
connection: '',
}),
),
expect(() =>
createDatabaseClient(
createConfig({
connection: '',
}),
),
).toThrowError();
});
it('throws an error without a connection', () => {
expect(
async () =>
await createDatabaseClient(
createConfig({
client: 'pg',
}),
),
expect(() =>
createDatabaseClient(
createConfig({
client: 'pg',
}),
),
).toThrowError();
});
});
packages/backend-common/src/database/connection.ts
+18 -2
View File
@@ -17,7 +17,7 @@
import knex from 'knex';
import { Config } from '@backstage/config';
import { mergeDatabaseConfig } from './config';
import { createPgDatabaseClient } from './postgres';
import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres';
import { createSqliteDatabaseClient } from './sqlite3';
type DatabaseClient = 'pg' | 'sqlite3' | string;
@@ -28,7 +28,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | string;
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export async function createDatabaseClient(
export function createDatabaseClient(
dbConfig: Config,
overrides?: Partial<knex.Config>,
) {
@@ -48,3 +48,19 @@ export async function createDatabaseClient(
* @deprecated Use createDatabaseClient instead
*/
export const createDatabase = createDatabaseClient;
/**
* Ensures that the given databases all exist, creating them if they do not.
*/
export async function ensureDatabaseExists(
dbConfig: Config,
...databases: Array<string>
) {
const client: DatabaseClient = dbConfig.getString('client');
if (client === 'pg') {
return ensurePgDatabaseExists(dbConfig, ...databases);
}
return undefined;
}
packages/backend-common/src/database/postgres.test.ts
+8 -9
View File
@@ -164,9 +164,9 @@ describe('postgres', () => {
});
describe(createPgDatabaseClient, () => {
it('creates a postgres knex instance', async () => {
it('creates a postgres knex instance', () => {
expect(
await createPgDatabaseClient(
createPgDatabaseClient(
createConfig({
host: 'acme',
user: 'foo',
@@ -177,14 +177,13 @@ describe('postgres', () => {
).toBeTruthy();
});
it('attempts to read an ssl cert', async () => {
expect(
async () =>
await createPgDatabaseClient(
createConfig(
'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
),
it('attempts to read an ssl cert', () => {
expect(() =>
createPgDatabaseClient(
createConfig(
'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
),
),
).toThrowError(/no such file or directory/);
});
});
packages/backend-common/src/database/postgres.ts
+44 -51
View File
@@ -15,37 +15,26 @@
*/
import knex, { PgConnectionConfig } from 'knex';
import { cloneDeep } from 'lodash';
import { Config, JsonValue } from '@backstage/config';
import { Config } from '@backstage/config';
import { mergeDatabaseConfig } from './config';
/**
* Creates a knex Postgres database connection
* Creates a knex postgres database connection
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export async function createPgDatabaseClient(
export function createPgDatabaseClient(
dbConfig: Config,
overrides?: knex.Config,
) {
const baseConfig = buildPgDatabaseConfig(dbConfig, overrides);
const knexConfig = baseConfig;
// Bootstrap the missing database.
if (!!baseConfig?.connection.database) {
const knexAdminConfig = buildPgDatabaseAdminConfig(cloneDeep(baseConfig));
const admin = knex(knexAdminConfig);
await ensurePgDatabase(admin, baseConfig.connection.database);
}
const knexConfig = buildPgDatabaseConfig(dbConfig, overrides);
const database = knex(knexConfig);
return database;
}
/**
* Builds a knex Postgres database connection
* Builds a knex postgres database connection
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
@@ -55,7 +44,7 @@ export function buildPgDatabaseConfig(
overrides?: knex.Config,
) {
return mergeDatabaseConfig(
cloneDeep(dbConfig.get()),
dbConfig.get(),
{
connection: getPgConnectionConfig(dbConfig, !!overrides),
useNullAsDefault: true,
@@ -65,23 +54,10 @@ export function buildPgDatabaseConfig(
}
/**
* Builds a knex Postgres database connection for database creation
* Gets the postgres connection config
*
* @param dbConfig The database config
*/
function buildPgDatabaseAdminConfig(dbConfig: JsonValue) {
return mergeDatabaseConfig(dbConfig, {
connection: {
database: 'postgres',
},
});
}
/**
* Gets the Postgres connection config
*
* @param dbConfig The database config
* @param parseConnectionString Flag to explicitly control connection string parsing
* @param parseConnectionString Flag to explictly control connection string parsing
*/
export function getPgConnectionConfig(
dbConfig: Config,
@@ -101,25 +77,6 @@ export function getPgConnectionConfig(
: connection;
}
/**
* Creates the missing Postgres database if it does not exist
*
* @param admin The administrative database connection, defaulting to the `postgres` database
* @param database The name of the database to create
*/
async function ensurePgDatabase(admin: knex, database: string) {
const result = await admin
.from('pg_database')
.where('datname', database)
.count<Record<string, { count: string }>>();
if (parseInt(result[0].count, 10) > 0) {
return;
}
await admin.raw(`CREATE DATABASE ??`, [database]);
}
/**
* Parses a connection string using pg-connection-string
*
@@ -138,3 +95,39 @@ function requirePgConnectionString() {
throw new Error(`${message}\n${e.message}`);
}
}
/**
* Creates the missing Postgres database if it does not exist
*
* @param dbConfig The database config
* @param databases The name of the databases to create
*/
export async function ensurePgDatabaseExists(
dbConfig: Config,
...databases: Array<string>
) {
const admin = createPgDatabaseClient(dbConfig, {
connection: {
database: 'postgres',
},
});
try {
const ensureDatabase = async (database: string) => {
const result = await admin
.from('pg_database')
.where('datname', database)
.count<Record<string, { count: string }>>();
if (parseInt(result[0].count, 10) > 0) {
return;
}
await admin.raw(`CREATE DATABASE ??`, [database]);
};
await Promise.all(databases.map(ensureDatabase));
} finally {
await admin.destroy();
}
}
packages/backend/src/index.ts
+30 -29
View File
@@ -24,6 +24,7 @@
import Router from 'express-promise-router';
import {
ensureDatabaseExists,
createDatabaseClient,
createServiceBuilder,
loadBackendConfig,
@@ -49,9 +50,9 @@ import { PluginEnvironment } from './types';
function makeCreateEnv(loadedConfigs: AppConfig[]) {
const config = ConfigReader.fromConfigs(loadedConfigs);
return async (plugin: string): Promise<PluginEnvironment> => {
return (plugin: string): PluginEnvironment => {
const logger = getRootLogger().child({ type: 'plugin', plugin });
const database = await createDatabaseClient(
const database = createDatabaseClient(
config.getConfig('backend.database'),
{
connection: {
@@ -68,41 +69,41 @@ async function main() {
const configs = await loadBackendConfig();
const configReader = ConfigReader.fromConfigs(configs);
const createEnv = makeCreateEnv(configs);
const healthcheckEnv = useHotMemoize(module, async () =>
createEnv('healthcheck'),
await ensureDatabaseExists(
configReader.getConfig('backend.database'),
'backstage_plugin_catalog',
'backstage_plugin_auth',
);
const catalogEnv = useHotMemoize(module, async () => createEnv('catalog'));
const scaffolderEnv = useHotMemoize(module, async () =>
createEnv('scaffolder'),
);
const authEnv = useHotMemoize(module, async () => createEnv('auth'));
const proxyEnv = useHotMemoize(module, async () => createEnv('proxy'));
const rollbarEnv = useHotMemoize(module, async () => createEnv('rollbar'));
const sentryEnv = useHotMemoize(module, async () => createEnv('sentry'));
const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs'));
const kubernetesEnv = useHotMemoize(module, async () =>
createEnv('kubernetes'),
);
const graphqlEnv = useHotMemoize(module, async () => createEnv('graphql'));
const appEnv = useHotMemoize(module, async () => createEnv('app'));
const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck'));
const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));
const authEnv = useHotMemoize(module, () => createEnv('auth'));
const proxyEnv = useHotMemoize(module, () => createEnv('proxy'));
const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar'));
const sentryEnv = useHotMemoize(module, () => createEnv('sentry'));
const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs'));
const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes'));
const graphqlEnv = useHotMemoize(module, () => createEnv('graphql'));
const appEnv = useHotMemoize(module, () => createEnv('app'));
const apiRouter = Router();
apiRouter.use('/catalog', await catalog(await catalogEnv));
apiRouter.use('/rollbar', await rollbar(await rollbarEnv));
apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv));
apiRouter.use('/sentry', await sentry(await sentryEnv));
apiRouter.use('/auth', await auth(await authEnv));
apiRouter.use('/techdocs', await techdocs(await techdocsEnv));
apiRouter.use('/kubernetes', await kubernetes(await kubernetesEnv));
apiRouter.use('/proxy', await proxy(await proxyEnv));
apiRouter.use('/graphql', await graphql(await graphqlEnv));
apiRouter.use('/catalog', await catalog(catalogEnv));
apiRouter.use('/rollbar', await rollbar(rollbarEnv));
apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv));
apiRouter.use('/sentry', await sentry(sentryEnv));
apiRouter.use('/auth', await auth(authEnv));
apiRouter.use('/techdocs', await techdocs(techdocsEnv));
apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv));
apiRouter.use('/proxy', await proxy(proxyEnv));
apiRouter.use('/graphql', await graphql(graphqlEnv));
apiRouter.use(notFoundHandler());
const service = createServiceBuilder(module)
.loadConfig(configReader)
.addRouter('', await healthcheck(await healthcheckEnv))
.addRouter('', await healthcheck(healthcheckEnv))
.addRouter('/api', apiRouter)
.addRouter('', await app(await appEnv));
.addRouter('', await app(appEnv));
await service.start().catch(err => {
console.log(err);
packages/create-app/templates/default-app/packages/backend/src/index.ts
+18 -12
View File
@@ -8,6 +8,7 @@
import Router from 'express-promise-router';
import {
ensureDatabaseExists,
createDatabaseClient,
createServiceBuilder,
loadBackendConfig,
@@ -27,9 +28,9 @@ import { PluginEnvironment } from './types';
function makeCreateEnv(loadedConfigs: AppConfig[]) {
const config = ConfigReader.fromConfigs(loadedConfigs);
return async (plugin: string): Promise<PluginEnvironment> => {
return (plugin: string): PluginEnvironment => {
const logger = getRootLogger().child({ type: 'plugin', plugin });
const database = await createDatabaseClient(
const database = createDatabaseClient(
config.getConfig('backend.database'),
{
connection: {
@@ -46,19 +47,24 @@ async function main() {
const configs = await loadBackendConfig();
const configReader = ConfigReader.fromConfigs(configs);
const createEnv = makeCreateEnv(configs);
await ensureDatabaseExists(
configReader.getConfig('backend.database'),
'backstage_plugin_catalog',
'backstage_plugin_auth',
);
const catalogEnv = useHotMemoize(module, async () => createEnv('catalog'));
const scaffolderEnv = useHotMemoize(module, async () => createEnv('scaffolder'));
const authEnv = useHotMemoize(module, async () => createEnv('auth'));
const proxyEnv = useHotMemoize(module, async () => createEnv('proxy'));
const techdocsEnv = useHotMemoize(module, async () => createEnv('techdocs'));
const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder'));
const authEnv = useHotMemoize(module, () => createEnv('auth'));
const proxyEnv = useHotMemoize(module, () => createEnv('proxy'));
const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs'));
const apiRouter = Router();
apiRouter.use('/catalog', await catalog(await catalogEnv))
apiRouter.use('/scaffolder', await scaffolder(await scaffolderEnv))
apiRouter.use('/auth', await auth(await authEnv))
apiRouter.use('/techdocs', await techdocs(await techdocsEnv))
apiRouter.use('/proxy', await proxy(await proxyEnv))
apiRouter.use('/catalog', await catalog(catalogEnv))
apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv))
apiRouter.use('/auth', await auth(authEnv))
apiRouter.use('/techdocs', await techdocs(techdocsEnv))
apiRouter.use('/proxy', await proxy(proxyEnv))
apiRouter.use(notFoundHandler());
const service = createServiceBuilder(module)