From 815983ae84658e302f02641c1857d60b18d2b20e Mon Sep 17 00:00:00 2001 From: Andrew Thauer <6507159+andrewthauer@users.noreply.github.com> Date: Thu, 13 Aug 2020 22:56:45 -0400 Subject: [PATCH] add shared database connection helpers --- app-config.yaml | 3 + packages/backend-common/package.json | 10 + .../src/database/config.test.ts | 97 +++++++++ .../backend-common/src/database/config.ts | 27 +++ .../src/database/connection.test.ts | 114 +++++++++++ .../backend-common/src/database/connection.ts | 41 ++++ packages/backend-common/src/database/index.ts | 20 ++ .../src/database/postgres.test.ts | 189 ++++++++++++++++++ .../backend-common/src/database/postgres.ts | 80 ++++++++ .../src/database/sqlite3.test.ts | 84 ++++++++ .../backend-common/src/database/sqlite3.ts | 58 ++++++ packages/backend-common/src/index.ts | 1 + packages/backend/package.json | 1 + packages/backend/src/index.ts | 38 +--- 14 files changed, 730 insertions(+), 33 deletions(-) create mode 100644 packages/backend-common/src/database/config.test.ts create mode 100644 packages/backend-common/src/database/config.ts create mode 100644 packages/backend-common/src/database/connection.test.ts create mode 100644 packages/backend-common/src/database/connection.ts create mode 100644 packages/backend-common/src/database/index.ts create mode 100644 packages/backend-common/src/database/postgres.test.ts create mode 100644 packages/backend-common/src/database/postgres.ts create mode 100644 packages/backend-common/src/database/sqlite3.test.ts create mode 100644 packages/backend-common/src/database/sqlite3.ts diff --git a/app-config.yaml b/app-config.yaml index 54684b10a9..b12fc50176 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -9,6 +9,9 @@ backend: origin: http://localhost:3000 methods: [GET, POST, PUT, DELETE] credentials: true + database: + client: sqlite3 + connection: ':memory:' proxy: '/circleci/api': diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index e21d0eb0d7..5cf2ee95e5 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -39,10 +39,20 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "helmet": "^4.0.0", + "knex": "^0.21.1", + "lodash": "^4.17.15", "morgan": "^1.10.0", "stoppable": "^1.1.0", "winston": "^3.2.1" }, + "peerDependencies": { + "pg-connection-string": "^2.3.0" + }, + "peerDependenciesMeta": { + "pg-connection-string": { + "optional": true + } + }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.18", "@types/compression": "^1.7.0", diff --git a/packages/backend-common/src/database/config.test.ts b/packages/backend-common/src/database/config.test.ts new file mode 100644 index 0000000000..acda0e47d8 --- /dev/null +++ b/packages/backend-common/src/database/config.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mergeDatabaseConfig } from './config'; + +describe('config', () => { + describe(mergeDatabaseConfig, () => { + it('does not modify the config', () => { + expect( + mergeDatabaseConfig( + { + client: 'pg', + connection: '', + useNullAsDefault: true, + }, + {}, + ), + ).toEqual({ + client: 'pg', + connection: '', + useNullAsDefault: true, + }); + }); + + it('does a deep merge', () => { + expect( + mergeDatabaseConfig( + { + client: 'pg', + connection: { + database: 'dbname', + ssl: { + ca: 'foo', + }, + }, + }, + { + connection: { + password: 'secret', + ssl: { + rejectUnauthorized: true, + }, + }, + pool: { min: 0, max: 7 }, + }, + ), + ).toEqual({ + client: 'pg', + connection: { + database: 'dbname', + password: 'secret', + ssl: { + rejectUnauthorized: true, + ca: 'foo', + }, + }, + pool: { min: 0, max: 7 }, + }); + }); + + it('merges string config objects', () => { + expect( + mergeDatabaseConfig( + { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }, + { + connection: { + filename: '/path/to/file', + }, + }, + ), + ).toEqual({ + client: 'sqlite3', + connection: { + filename: '/path/to/file', + }, + useNullAsDefault: true, + }); + }); + }); +}); diff --git a/packages/backend-common/src/database/config.ts b/packages/backend-common/src/database/config.ts new file mode 100644 index 0000000000..80abc06aa2 --- /dev/null +++ b/packages/backend-common/src/database/config.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { merge } from 'lodash'; + +/** + * Merges database objects together + * + * @param config The base config + * @param overrides Any additional overrides + */ +export function mergeDatabaseConfig(config: any, ...overrides: any[]) { + return merge(config, ...overrides); +} diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts new file mode 100644 index 0000000000..814503fda1 --- /dev/null +++ b/packages/backend-common/src/database/connection.test.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { createDatabase } from './connection'; + +describe('database connection', () => { + const createConfig = (data: any) => + ConfigReader.fromConfigs([ + { + context: '', + data, + }, + ]); + + describe(createDatabase, () => { + it('returns a postgres connection', () => { + expect( + createDatabase( + createConfig({ + client: 'pg', + connection: { + host: 'acme', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }), + ), + ).toBeTruthy(); + }); + + it('returns an sqlite connection', () => { + expect( + createDatabase( + createConfig({ + client: 'sqlite3', + connection: ':memory:', + }), + ), + ).toBeTruthy(); + }); + + it('tries to create a mysql connection as a passthrough', () => { + expect(() => + createDatabase( + createConfig({ + client: 'mysql', + connection: { + host: '127.0.0.1', + user: 'foo', + password: 'bar', + database: 'dbname', + }, + }), + ), + ).toThrowError(/Cannot find module 'mysql'/); + }); + + it('accepts overrides', () => { + expect( + createDatabase( + createConfig({ + client: 'pg', + connection: { + host: 'acme', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }), + { + connection: { + database: 'foo', + }, + }, + ), + ).toBeTruthy(); + }); + + it('throws an error without a client', () => { + expect(() => + createDatabase( + createConfig({ + connection: '', + }), + ), + ).toThrowError(); + }); + + it('throws an error without a connection', () => { + expect(() => + createDatabase( + createConfig({ + client: 'pg', + }), + ), + ).toThrowError(); + }); + }); +}); diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts new file mode 100644 index 0000000000..bbb1752737 --- /dev/null +++ b/packages/backend-common/src/database/connection.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; +import { ConfigReader } from '@backstage/config'; +import { mergeDatabaseConfig } from './config'; +import { createPgDatabase } from './postgres'; +import { createSqlite3Database } from './sqlite3'; + +type DatabaseClient = 'pg' | 'sqlite3' | string; + +/** + * Creates a knex database connection + * + * @param config The database config + * @param overrides Additional options to merge with the config + */ +export function createDatabase(config: ConfigReader, overrides?: knex.Config) { + const client: DatabaseClient = config.getString('client'); + + if (client === 'pg') { + return createPgDatabase(config, overrides); + } else if (client === 'sqlite3') { + return createSqlite3Database(config, overrides); + } + + return knex(mergeDatabaseConfig(config.get(), overrides)); +} diff --git a/packages/backend-common/src/database/index.ts b/packages/backend-common/src/database/index.ts new file mode 100644 index 0000000000..641adc565c --- /dev/null +++ b/packages/backend-common/src/database/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './config'; +export * from './connection'; +export * from './postgres'; +export * from './sqlite3'; diff --git a/packages/backend-common/src/database/postgres.test.ts b/packages/backend-common/src/database/postgres.test.ts new file mode 100644 index 0000000000..8edfc2c15d --- /dev/null +++ b/packages/backend-common/src/database/postgres.test.ts @@ -0,0 +1,189 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { + parsePgConnectionString, + buildPgDatabaseConfig, + createPgDatabase, +} from './postgres'; + +describe('postgres', () => { + const createConfig = (connection: any) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + client: 'pg', + connection, + }, + }, + ]); + + describe(buildPgDatabaseConfig, () => { + it('builds a postgres config', () => { + expect( + buildPgDatabaseConfig( + createConfig({ + host: 'acme', + user: 'foo', + password: 'bar', + port: '5432', + database: 'foodb', + }), + ), + ).toEqual({ + client: 'pg', + connection: { + host: 'acme', + user: 'foo', + password: 'bar', + port: '5432', + database: 'foodb', + }, + useNullAsDefault: true, + }); + }); + + it('builds a connection string config', () => { + expect( + buildPgDatabaseConfig( + createConfig('postgresql://foo:bar@acme:5432/foodb'), + ), + ).toEqual({ + client: 'pg', + connection: 'postgresql://foo:bar@acme:5432/foodb', + useNullAsDefault: true, + }); + }); + + it('overrides the database name', () => { + expect( + buildPgDatabaseConfig( + createConfig({ + host: 'somehost', + user: 'postgres', + password: 'pass', + database: 'foo', + }), + { connection: { database: 'foodb' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + host: 'somehost', + user: 'postgres', + password: 'pass', + database: 'foodb', + }, + useNullAsDefault: true, + }); + }); + + it('adds additional config settings', () => { + expect( + buildPgDatabaseConfig( + createConfig({ + host: 'somehost', + user: 'postgres', + password: 'pass', + database: 'foo', + }), + { + connection: { database: 'foodb' }, + pool: { min: 0, max: 7 }, + debug: true, + }, + ), + ).toEqual({ + client: 'pg', + connection: { + host: 'somehost', + user: 'postgres', + password: 'pass', + database: 'foodb', + }, + useNullAsDefault: true, + pool: { min: 0, max: 7 }, + debug: true, + }); + }); + + it('overrides the database from connection string', () => { + expect( + buildPgDatabaseConfig( + createConfig('postgresql://postgres:pass@localhost:5432/dbname'), + { connection: { database: 'foodb' } }, + ), + ).toEqual({ + client: 'pg', + connection: { + host: 'localhost', + user: 'postgres', + password: 'pass', + port: '5432', + database: 'foodb', + }, + useNullAsDefault: true, + }); + }); + }); + + describe(createPgDatabase, () => { + it('creates a postgres knex instance', () => { + expect( + createPgDatabase( + createConfig({ + client: 'pg', + connection: { + host: 'acme', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }), + ), + ).toBeTruthy(); + }); + + it('attempts to read an ssl cert', () => { + expect(() => + createPgDatabase( + createConfig( + 'postgresql://postgres:pass@localhost:5432/dbname?sslrootcert=/path/to/file', + ), + ), + ).toThrowError(/no such file or directory/); + }); + }); + + describe(parsePgConnectionString, () => { + it('parses a connection string uri ', () => { + expect( + parsePgConnectionString( + 'postgresql://postgres:pass@foobar:5432/dbname?ssl=true', + ), + ).toEqual({ + host: 'foobar', + user: 'postgres', + password: 'pass', + port: '5432', + database: 'dbname', + ssl: true, + }); + }); + }); +}); diff --git a/packages/backend-common/src/database/postgres.ts b/packages/backend-common/src/database/postgres.ts new file mode 100644 index 0000000000..09b6d0653b --- /dev/null +++ b/packages/backend-common/src/database/postgres.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; +import { ConfigReader } from '@backstage/config'; +import { mergeDatabaseConfig } from './config'; + +/** + * Creates a knex sqlite3 database connection + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function createPgDatabase( + dbConfig: ConfigReader, + overrides?: knex.Config, +) { + const knexConfig = buildPgDatabaseConfig(dbConfig, overrides); + const database = knex(knexConfig); + return database; +} + +/** + * Builds a knex postgres database connection + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function buildPgDatabaseConfig( + dbConfig: ConfigReader, + overrides?: knex.Config, +) { + const connection = dbConfig.get('connection') as any; + + return mergeDatabaseConfig( + dbConfig.get(), + { + // Only parse the connection string when overrides are provided + connection: + overrides && + (typeof connection === 'string' || connection instanceof String) + ? parsePgConnectionString(connection as string) + : connection, + useNullAsDefault: true, + }, + overrides, + ); +} + +/** + * Parses a connection string using pg-connection-string + * + * @param connectionString The postgres connection string + */ +export function parsePgConnectionString(connectionString: string) { + const parse = requirePgConnectionString(); + return parse(connectionString); +} + +function requirePgConnectionString() { + try { + return require('pg-connection-string').parse; + } catch (e) { + const message = `Postgres: Install 'pg-connection-string'`; + throw new Error(`${message}\n${e.message}`); + } +} diff --git a/packages/backend-common/src/database/sqlite3.test.ts b/packages/backend-common/src/database/sqlite3.test.ts new file mode 100644 index 0000000000..a25720b81c --- /dev/null +++ b/packages/backend-common/src/database/sqlite3.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { buildSqlite3DatabaseConfig, createSqlite3Database } from './sqlite3'; + +describe('sqlite3', () => { + const createConfig = (connection: any) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + client: 'sqlite3', + connection, + }, + }, + ]); + + describe(buildSqlite3DatabaseConfig, () => { + it('buidls a string connection', () => { + expect(buildSqlite3DatabaseConfig(createConfig(':memory:'))).toEqual({ + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + }); + + it('builds a filename connection', () => { + expect( + buildSqlite3DatabaseConfig( + createConfig({ + filename: '/path/to/foo', + }), + ), + ).toEqual({ + client: 'sqlite3', + connection: { + filename: '/path/to/foo', + }, + useNullAsDefault: true, + }); + }); + + it('replaces the connection with an override', () => { + expect( + buildSqlite3DatabaseConfig(createConfig(':memory:'), { + connection: { filename: '/path/to/foo' }, + }), + ).toEqual({ + client: 'sqlite3', + connection: { + filename: '/path/to/foo', + }, + useNullAsDefault: true, + }); + }); + }); + + describe(createSqlite3Database, () => { + it('creates an in memory knex instance', () => { + expect( + createSqlite3Database( + createConfig({ + client: 'sqlite3', + connection: ':memory:', + }), + ), + ).toBeTruthy(); + }); + }); +}); diff --git a/packages/backend-common/src/database/sqlite3.ts b/packages/backend-common/src/database/sqlite3.ts new file mode 100644 index 0000000000..6249394d05 --- /dev/null +++ b/packages/backend-common/src/database/sqlite3.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knex from 'knex'; +import { ConfigReader } from '@backstage/config'; +import { mergeDatabaseConfig } from './config'; + +/** + * Creates a knex sqlite3 database connection + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function createSqlite3Database( + dbConfig: ConfigReader, + overrides?: knex.Config, +) { + const knexConfig = buildSqlite3DatabaseConfig(dbConfig, overrides); + const database = knex(knexConfig); + + database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }); + + return database; +} + +/** + * Builds a knex sqlite3 connection config + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function buildSqlite3DatabaseConfig( + dbConfig: ConfigReader, + overrides?: knex.Config, +) { + return mergeDatabaseConfig( + dbConfig.get(), + { + useNullAsDefault: true, + }, + overrides, + ); +} diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index a3f57ed9df..f499613851 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -15,6 +15,7 @@ */ export * from './config'; +export * from './database'; export * from './errors'; export * from './logging'; export * from './middleware'; diff --git a/packages/backend/package.json b/packages/backend/package.json index 6e1bcdaf69..c80fbc7c38 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -35,6 +35,7 @@ "express": "^4.17.1", "knex": "^0.21.1", "pg": "^8.3.0", + "pg-connection-string": "^2.3.0", "sqlite3": "^4.2.0", "winston": "^3.2.1" }, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 261a4d2894..cb24b00477 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -23,13 +23,13 @@ */ import { + createDatabase, createServiceBuilder, loadBackendConfig, getRootLogger, useHotMemoize, } from '@backstage/backend-common'; import { ConfigReader, AppConfig } from '@backstage/config'; -import knex, { PgConnectionConfig } from 'knex'; import healthcheck from './plugins/healthcheck'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; @@ -47,38 +47,10 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) { return (plugin: string): PluginEnvironment => { const logger = getRootLogger().child({ type: 'plugin', plugin }); - // Supported DBs are sqlite and postgres - const isPg = [ - 'POSTGRES_USER', - 'POSTGRES_HOST', - 'POSTGRES_PASSWORD', - ].every(key => config.getOptional(`backend.${key}`)); - - let knexConfig; - - if (isPg) { - knexConfig = { - client: 'pg', - useNullAsDefault: true, - connection: { - port: config.getOptionalNumber('backend.POSTGRES_PORT'), - host: config.getString('backend.POSTGRES_HOST'), - user: config.getString('backend.POSTGRES_USER'), - password: config.getString('backend.POSTGRES_PASSWORD'), - database: `backstage_plugin_${plugin}`, - } as PgConnectionConfig, - }; - } else { - knexConfig = { - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }; - } - - const database = knex(knexConfig); - database.client.pool.on('createSuccess', (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); + const database = createDatabase(config.getConfig('backend.database'), { + connection: { + database: `backstage_plugin_${plugin}`, + }, }); return { logger, database, config }; };