paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Merge pull request #1915 from andrewthauer/common-db-utils

Browse Source 
Add shared database connection helpers
This commit is contained in:
Patrik Oldsberg
2020-08-15 13:57:44 +02:00
committed by GitHub
parent dca45f28af 4285289b5a
commit fe48229005
14 changed files with 744 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+3
View File
@@ -11,6 +11,9 @@ backend:
origin: http://localhost:3000
methods: [GET, POST, PUT, DELETE]
credentials: true
database:
client: sqlite3
connection: ':memory:'
proxy:
'/circleci/api':
packages/backend-common/package.json
+10
View File
@@ -39,10 +39,20 @@
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"helmet": "^4.0.0",
"knex": "^0.21.1",
"lodash": "^4.17.15",
"morgan": "^1.10.0",
"stoppable": "^1.1.0",
"winston": "^3.2.1"
},
"peerDependencies": {
"pg-connection-string": "^2.3.0"
},
"peerDependenciesMeta": {
"pg-connection-string": {
"optional": true
}
},
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.18",
"@types/compression": "^1.7.0",
packages/backend-common/src/database/config.test.ts
+111
View File
@@ -0,0 +1,111 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { mergeDatabaseConfig } from './config';
describe('config', () => {
describe(mergeDatabaseConfig, () => {
it('does not require overrides', () => {
expect(
mergeDatabaseConfig({
client: 'pg',
connection: '',
useNullAsDefault: true,
}),
).toEqual({
client: 'pg',
connection: '',
useNullAsDefault: true,
});
});
it('accepts an empty object', () => {
expect(
mergeDatabaseConfig(
{
client: 'pg',
connection: '',
useNullAsDefault: true,
},
{},
),
).toEqual({
client: 'pg',
connection: '',
useNullAsDefault: true,
});
});
it('does a deep merge', () => {
expect(
mergeDatabaseConfig(
{
client: 'pg',
connection: {
database: 'dbname',
ssl: {
ca: 'foo',
},
},
},
{
connection: {
password: 'secret',
ssl: {
rejectUnauthorized: true,
},
},
pool: { min: 0, max: 7 },
},
),
).toEqual({
client: 'pg',
connection: {
database: 'dbname',
password: 'secret',
ssl: {
rejectUnauthorized: true,
ca: 'foo',
},
},
pool: { min: 0, max: 7 },
});
});
it('replaces a string connection', () => {
expect(
mergeDatabaseConfig(
{
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
},
{
connection: {
filename: '/path/to/file',
},
},
),
).toEqual({
client: 'sqlite3',
connection: {
filename: '/path/to/file',
},
useNullAsDefault: true,
});
});
});
});
packages/backend-common/src/database/config.ts
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { merge } from 'lodash';
/**
* Merges database objects together
*
* @param config The base config
* @param overrides Any additional overrides
*/
export function mergeDatabaseConfig(config: any, ...overrides: any[]) {
return merge(config, ...overrides);
}
packages/backend-common/src/database/connection.test.ts
+114
View File
@@ -0,0 +1,114 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { createDatabase } from './connection';
describe('database connection', () => {
const createConfig = (data: any) =>
ConfigReader.fromConfigs([
{
context: '',
data,
},
]);
describe(createDatabase, () => {
it('returns a postgres connection', () => {
expect(
createDatabase(
createConfig({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
database: 'foodb',
},
}),
),
).toBeTruthy();
});
it('returns an sqlite connection', () => {
expect(
createDatabase(
createConfig({
client: 'sqlite3',
connection: ':memory:',
}),
),
).toBeTruthy();
});
it('tries to create a mysql connection as a passthrough', () => {
expect(() =>
createDatabase(
createConfig({
client: 'mysql',
connection: {
host: '127.0.0.1',
user: 'foo',
password: 'bar',
database: 'dbname',
},
}),
),
).toThrowError(/Cannot find module 'mysql'/);
});
it('accepts overrides', () => {
expect(
createDatabase(
createConfig({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
database: 'foodb',
},
}),
{
connection: {
database: 'foo',
},
},
),
).toBeTruthy();
});
it('throws an error without a client', () => {
expect(() =>
createDatabase(
createConfig({
connection: '',
}),
),
).toThrowError();
});
it('throws an error without a connection', () => {
expect(() =>
createDatabase(
createConfig({
client: 'pg',
}),
),
).toThrowError();
});
});
});
packages/backend-common/src/database/connection.ts
+44
View File
@@ -0,0 +1,44 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knex from 'knex';
import { ConfigReader } from '@backstage/config';
import { mergeDatabaseConfig } from './config';
import { createPgDatabase } from './postgres';
import { createSqlite3Database } from './sqlite3';
type DatabaseClient = 'pg' | 'sqlite3' | string;
/**
* Creates a knex database connection
*
* @param config The database config
* @param overrides Additional options to merge with the config
*/
export function createDatabase(
config: ConfigReader,
overrides?: Partial<knex.Config>,
) {
const client: DatabaseClient = config.getString('client');
if (client === 'pg') {
return createPgDatabase(config, overrides);
} else if (client === 'sqlite3') {
return createSqlite3Database(config, overrides);
}
return knex(mergeDatabaseConfig(config.get(), overrides));
}
packages/backend-common/src/database/index.ts
+17
View File
@@ -0,0 +1,17 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './connection';
packages/backend-common/src/database/postgres.test.ts
+189
View File
@@ -0,0 +1,189 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
parsePgConnectionString,
buildPgDatabaseConfig,
createPgDatabase,
} from './postgres';
describe('postgres', () => {
const createConfig = (connection: any) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
client: 'pg',
connection,
},
},
]);
describe(buildPgDatabaseConfig, () => {
it('builds a postgres config', () => {
expect(
buildPgDatabaseConfig(
createConfig({
host: 'acme',
user: 'foo',
password: 'bar',
port: '5432',
database: 'foodb',
}),
),
).toEqual({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
port: '5432',
database: 'foodb',
},
useNullAsDefault: true,
});
});
it('builds a connection string config', () => {
expect(
buildPgDatabaseConfig(
createConfig('postgresql://foo:bar@acme:5432/foodb'),
),
).toEqual({
client: 'pg',
connection: 'postgresql://foo:bar@acme:5432/foodb',
useNullAsDefault: true,
});
});
it('overrides the database name', () => {
expect(
buildPgDatabaseConfig(
createConfig({
host: 'somehost',
user: 'postgres',
password: 'pass',
database: 'foo',
}),
{ connection: { database: 'foodb' } },
),
).toEqual({
client: 'pg',
connection: {
host: 'somehost',
user: 'postgres',
password: 'pass',
database: 'foodb',
},
useNullAsDefault: true,
});
});
it('adds additional config settings', () => {
expect(
buildPgDatabaseConfig(
createConfig({
host: 'somehost',
user: 'postgres',
password: 'pass',
database: 'foo',
}),
{
connection: { database: 'foodb' },
pool: { min: 0, max: 7 },
debug: true,
},
),
).toEqual({
client: 'pg',
connection: {
host: 'somehost',
user: 'postgres',
password: 'pass',
database: 'foodb',
},
useNullAsDefault: true,
pool: { min: 0, max: 7 },
debug: true,
});
});
it('overrides the database from connection string', () => {
expect(
buildPgDatabaseConfig(
createConfig('postgresql://postgres:pass@localhost:5432/dbname'),
{ connection: { database: 'foodb' } },
),
).toEqual({
client: 'pg',
connection: {
host: 'localhost',
user: 'postgres',
password: 'pass',
port: '5432',
database: 'foodb',
},
useNullAsDefault: true,
});
});
});
describe(createPgDatabase, () => {
it('creates a postgres knex instance', () => {
expect(
createPgDatabase(
createConfig({
client: 'pg',
connection: {
host: 'acme',
user: 'foo',
password: 'bar',
database: 'foodb',
},
}),
),
).toBeTruthy();
});
it('attempts to read an ssl cert', () => {
expect(() =>
createPgDatabase(
createConfig(
'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file',
),
),
).toThrowError(/no such file or directory/);
});
});
describe(parsePgConnectionString, () => {
it('parses a connection string uri ', () => {
expect(
parsePgConnectionString(
'postgresql://postgres:pass@foobar:5432/dbname?ssl=true',
),
).toEqual({
host: 'foobar',
user: 'postgres',
password: 'pass',
port: '5432',
database: 'dbname',
ssl: true,
});
});
});
});
packages/backend-common/src/database/postgres.ts
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knex from 'knex';
import { ConfigReader } from '@backstage/config';
import { mergeDatabaseConfig } from './config';
/**
* Creates a knex sqlite3 database connection
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export function createPgDatabase(
dbConfig: ConfigReader,
overrides?: knex.Config,
) {
const knexConfig = buildPgDatabaseConfig(dbConfig, overrides);
const database = knex(knexConfig);
return database;
}
/**
* Builds a knex postgres database connection
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export function buildPgDatabaseConfig(
dbConfig: ConfigReader,
overrides?: knex.Config,
) {
const connection = dbConfig.get('connection') as any;
return mergeDatabaseConfig(
dbConfig.get(),
{
// Only parse the connection string when overrides are provided
connection:
overrides &&
(typeof connection === 'string' || connection instanceof String)
? parsePgConnectionString(connection as string)
: connection,
useNullAsDefault: true,
},
overrides,
);
}
/**
* Parses a connection string using pg-connection-string
*
* @param connectionString The postgres connection string
*/
export function parsePgConnectionString(connectionString: string) {
const parse = requirePgConnectionString();
return parse(connectionString);
}
function requirePgConnectionString() {
try {
return require('pg-connection-string').parse;
} catch (e) {
const message = `Postgres: Install 'pg-connection-string'`;
throw new Error(`${message}\n${e.message}`);
}
}
packages/backend-common/src/database/sqlite3.test.ts
+84
View File
@@ -0,0 +1,84 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { buildSqlite3DatabaseConfig, createSqlite3Database } from './sqlite3';
describe('sqlite3', () => {
const createConfig = (connection: any) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
client: 'sqlite3',
connection,
},
},
]);
describe(buildSqlite3DatabaseConfig, () => {
it('buidls a string connection', () => {
expect(buildSqlite3DatabaseConfig(createConfig(':memory:'))).toEqual({
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
});
});
it('builds a filename connection', () => {
expect(
buildSqlite3DatabaseConfig(
createConfig({
filename: '/path/to/foo',
}),
),
).toEqual({
client: 'sqlite3',
connection: {
filename: '/path/to/foo',
},
useNullAsDefault: true,
});
});
it('replaces the connection with an override', () => {
expect(
buildSqlite3DatabaseConfig(createConfig(':memory:'), {
connection: { filename: '/path/to/foo' },
}),
).toEqual({
client: 'sqlite3',
connection: {
filename: '/path/to/foo',
},
useNullAsDefault: true,
});
});
});
describe(createSqlite3Database, () => {
it('creates an in memory knex instance', () => {
expect(
createSqlite3Database(
createConfig({
client: 'sqlite3',
connection: ':memory:',
}),
),
).toBeTruthy();
});
});
});
packages/backend-common/src/database/sqlite3.ts
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knex from 'knex';
import { ConfigReader } from '@backstage/config';
import { mergeDatabaseConfig } from './config';
/**
* Creates a knex sqlite3 database connection
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export function createSqlite3Database(
dbConfig: ConfigReader,
overrides?: knex.Config,
) {
const knexConfig = buildSqlite3DatabaseConfig(dbConfig, overrides);
const database = knex(knexConfig);
database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
return database;
}
/**
* Builds a knex sqlite3 connection config
*
* @param dbConfig The database config
* @param overrides Additional options to merge with the config
*/
export function buildSqlite3DatabaseConfig(
dbConfig: ConfigReader,
overrides?: knex.Config,
) {
return mergeDatabaseConfig(
dbConfig.get(),
{
useNullAsDefault: true,
},
overrides,
);
}
packages/backend-common/src/index.ts
+1
View File
@@ -15,6 +15,7 @@
*/
export * from './config';
export * from './database';
export * from './errors';
export * from './logging';
export * from './middleware';
packages/backend/package.json
+1
View File
@@ -35,6 +35,7 @@
"express": "^4.17.1",
"knex": "^0.21.1",
"pg": "^8.3.0",
"pg-connection-string": "^2.3.0",
"sqlite3": "^4.2.0",
"winston": "^3.2.1"
},
packages/backend/src/index.ts
+5 -33
View File
@@ -23,13 +23,13 @@
*/
import {
createDatabase,
createServiceBuilder,
loadBackendConfig,
getRootLogger,
useHotMemoize,
} from '@backstage/backend-common';
import { ConfigReader, AppConfig } from '@backstage/config';
import knex, { PgConnectionConfig } from 'knex';
import healthcheck from './plugins/healthcheck';
import auth from './plugins/auth';
import catalog from './plugins/catalog';
@@ -47,38 +47,10 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
return (plugin: string): PluginEnvironment => {
const logger = getRootLogger().child({ type: 'plugin', plugin });
// Supported DBs are sqlite and postgres
const isPg = [
'POSTGRES_USER',
'POSTGRES_HOST',
'POSTGRES_PASSWORD',
].every(key => config.getOptional(`backend.${key}`));
let knexConfig;
if (isPg) {
knexConfig = {
client: 'pg',
useNullAsDefault: true,
connection: {
port: config.getOptionalNumber('backend.POSTGRES_PORT'),
host: config.getString('backend.POSTGRES_HOST'),
user: config.getString('backend.POSTGRES_USER'),
password: config.getString('backend.POSTGRES_PASSWORD'),
database: `backstage_plugin_${plugin}`,
} as PgConnectionConfig,
};
} else {
knexConfig = {
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
};
}
const database = knex(knexConfig);
database.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
const database = createDatabase(config.getConfig('backend.database'), {
connection: {
database: `backstage_plugin_${plugin}`,
},
});
return { logger, database, config };
};