From f9fb4a205e206a678a00f6cfcba4a6ebf226a46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 9 May 2021 22:42:56 +0200 Subject: [PATCH] Prep work for mysql support in backend-common MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/mighty-poets-cheer.md | 5 + .github/styles/vocab.txt | 1 + packages/backend-common/package.json | 4 +- .../src/database/connection.test.ts | 6 +- .../backend-common/src/database/connection.ts | 7 +- .../backend-common/src/database/mysql.test.ts | 188 ++++++++++++++++++ packages/backend-common/src/database/mysql.ts | 161 +++++++++++++++ yarn.lock | 51 ++++- 8 files changed, 416 insertions(+), 7 deletions(-) create mode 100644 .changeset/mighty-poets-cheer.md create mode 100644 packages/backend-common/src/database/mysql.test.ts create mode 100644 packages/backend-common/src/database/mysql.ts diff --git a/.changeset/mighty-poets-cheer.md b/.changeset/mighty-poets-cheer.md new file mode 100644 index 0000000000..47538489e4 --- /dev/null +++ b/.changeset/mighty-poets-cheer.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Prep work for mysql support in backend-common diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 694226b8cc..f95eeb6f89 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -177,6 +177,7 @@ mkdocs monorepo monorepos msw +mysql namespace namespaced namespaces diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index a144f1c23f..6b47cd9599 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -62,7 +62,8 @@ "stoppable": "^1.1.0", "tar": "^6.0.5", "unzipper": "^0.10.11", - "winston": "^3.2.1" + "winston": "^3.2.1", + "yn": "^4.0.0" }, "peerDependencies": { "pg-connection-string": "^2.3.0" @@ -94,6 +95,7 @@ "jest": "^26.0.1", "mock-fs": "^4.13.0", "msw": "^0.21.2", + "mysql2": "^2.2.5", "recursive-readdir": "^2.2.2", "supertest": "^6.1.3" }, diff --git a/packages/backend-common/src/database/connection.test.ts b/packages/backend-common/src/database/connection.test.ts index 9810ff9bef..6ab163ff7f 100644 --- a/packages/backend-common/src/database/connection.test.ts +++ b/packages/backend-common/src/database/connection.test.ts @@ -46,11 +46,11 @@ describe('database connection', () => { ).toBeTruthy(); }); - it('tries to create a mysql connection as a passthrough', () => { + it('returns a mysql connection', () => { expect(() => createDatabaseClient( new ConfigReader({ - client: 'mysql', + client: 'mysql2', connection: { host: '127.0.0.1', user: 'foo', @@ -59,7 +59,7 @@ describe('database connection', () => { }, }), ), - ).toThrowError(/Cannot find module 'mysql'/); + ).toBeTruthy(); }); it('accepts overrides', () => { diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index 3502e21674..fb16915cf4 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -14,9 +14,10 @@ * limitations under the License. */ -import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; +import knexFactory, { Knex } from 'knex'; import { mergeDatabaseConfig } from './config'; +import { createMysqlDatabaseClient, ensureMysqlDatabaseExists } from './mysql'; import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres'; import { createSqliteDatabaseClient } from './sqlite3'; @@ -36,6 +37,8 @@ export function createDatabaseClient( if (client === 'pg') { return createPgDatabaseClient(dbConfig, overrides); + } else if (client === 'mysql' || client === 'mysql2') { + return createMysqlDatabaseClient(dbConfig, overrides); } else if (client === 'sqlite3') { return createSqliteDatabaseClient(dbConfig, overrides); } @@ -60,6 +63,8 @@ export async function ensureDatabaseExists( if (client === 'pg') { return ensurePgDatabaseExists(dbConfig, ...databases); + } else if (client === 'mysql' || client === 'mysql2') { + return ensureMysqlDatabaseExists(dbConfig, ...databases); } return undefined; diff --git a/packages/backend-common/src/database/mysql.test.ts b/packages/backend-common/src/database/mysql.test.ts new file mode 100644 index 0000000000..9e23585d5b --- /dev/null +++ b/packages/backend-common/src/database/mysql.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + buildMysqlDatabaseConfig, + createMysqlDatabaseClient, + getMysqlConnectionConfig, + parseMysqlConnectionString, +} from './mysql'; + +describe('mysql', () => { + const createMockConnection = () => ({ + host: 'acme', + user: 'foo', + password: 'bar', + database: 'foodb', + }); + + const createMockConnectionString = () => 'mysql://foo:bar@acme:3306/foodb'; + + const createConfig = (connection: any): Config => + new ConfigReader({ client: 'mysql2', connection }); + + describe('buildMysqlDatabaseConfig', () => { + it('builds a mysql config', () => { + const mockConnection = createMockConnection(); + + expect(buildMysqlDatabaseConfig(createConfig(mockConnection))).toEqual({ + client: 'mysql2', + connection: mockConnection, + useNullAsDefault: true, + }); + }); + + it('builds a connection string config', () => { + const mockConnectionString = createMockConnectionString(); + + expect( + buildMysqlDatabaseConfig(createConfig(mockConnectionString)), + ).toEqual({ + client: 'mysql2', + connection: mockConnectionString, + useNullAsDefault: true, + }); + }); + + it('overrides the database name', () => { + const mockConnection = createMockConnection(); + + expect( + buildMysqlDatabaseConfig(createConfig(mockConnection), { + connection: { database: 'other_db' }, + }), + ).toEqual({ + client: 'mysql2', + connection: { + ...mockConnection, + database: 'other_db', + }, + useNullAsDefault: true, + }); + }); + + it('adds additional config settings', () => { + const mockConnection = createMockConnection(); + + expect( + buildMysqlDatabaseConfig(createConfig(mockConnection), { + connection: { database: 'other_db' }, + pool: { min: 0, max: 7 }, + debug: true, + }), + ).toEqual({ + client: 'mysql2', + connection: { + ...mockConnection, + database: 'other_db', + }, + useNullAsDefault: true, + pool: { min: 0, max: 7 }, + debug: true, + }); + }); + + it('overrides the database from connection string', () => { + const mockConnectionString = createMockConnectionString(); + const mockConnection = createMockConnection(); + + expect( + buildMysqlDatabaseConfig(createConfig(mockConnectionString), { + connection: { database: 'other_db' }, + }), + ).toEqual({ + client: 'mysql2', + connection: { + ...mockConnection, + port: 3306, + database: 'other_db', + }, + useNullAsDefault: true, + }); + }); + }); + + describe('getMysqlConnectionConfig', () => { + it('returns the connection object back', () => { + const mockConnection = createMockConnection(); + const config = createConfig(mockConnection); + + expect(getMysqlConnectionConfig(config)).toEqual(mockConnection); + }); + + it('does not parse the connection string', () => { + const mockConnection = createMockConnection(); + const config = createConfig(mockConnection); + + expect(getMysqlConnectionConfig(config, true)).toEqual(mockConnection); + }); + + it('automatically parses the connection string', () => { + const mockConnection = createMockConnection(); + const mockConnectionString = createMockConnectionString(); + const config = createConfig(mockConnectionString); + + expect(getMysqlConnectionConfig(config)).toEqual({ + ...mockConnection, + port: 3306, + }); + }); + + it('parses the connection string', () => { + const mockConnection = createMockConnection(); + const mockConnectionString = createMockConnectionString(); + const config = createConfig(mockConnectionString); + + expect(getMysqlConnectionConfig(config, true)).toEqual({ + ...mockConnection, + port: 3306, + }); + }); + }); + + describe('createMysqlDatabaseClient', () => { + it('creates a mysql knex instance', () => { + expect( + createMysqlDatabaseClient( + createConfig({ + host: 'acme', + user: 'foo', + password: 'bar', + database: 'foodb', + }), + ), + ).toBeTruthy(); + }); + }); + + describe('parseMysqlConnectionString', () => { + it('parses a connection string uri', () => { + expect( + parseMysqlConnectionString( + 'mysql://u:pass@foobar:3307/dbname?ssl=require', + ), + ).toEqual({ + host: 'foobar', + user: 'u', + password: 'pass', + port: 3307, + database: 'dbname', + ssl: 'require', + }); + }); + }); +}); diff --git a/packages/backend-common/src/database/mysql.ts b/packages/backend-common/src/database/mysql.ts new file mode 100644 index 0000000000..be4632baf6 --- /dev/null +++ b/packages/backend-common/src/database/mysql.ts @@ -0,0 +1,161 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import knexFactory, { Knex } from 'knex'; +import { mergeDatabaseConfig } from './config'; +import yn from 'yn'; + +/** + * Creates a knex mysql database connection + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function createMysqlDatabaseClient( + dbConfig: Config, + overrides?: Knex.Config, +) { + const knexConfig = buildMysqlDatabaseConfig(dbConfig, overrides); + const database = knexFactory(knexConfig); + return database; +} + +/** + * Builds a knex mysql database connection + * + * @param dbConfig The database config + * @param overrides Additional options to merge with the config + */ +export function buildMysqlDatabaseConfig( + dbConfig: Config, + overrides?: Knex.Config, +) { + return mergeDatabaseConfig( + dbConfig.get(), + { + connection: getMysqlConnectionConfig(dbConfig, !!overrides), + useNullAsDefault: true, + }, + overrides, + ); +} + +/** + * Gets the mysql connection config + * + * @param dbConfig The database config + * @param parseConnectionString Flag to explicitly control connection string parsing + */ +export function getMysqlConnectionConfig( + dbConfig: Config, + parseConnectionString?: boolean, +): Knex.MySqlConnectionConfig | string { + const connection = dbConfig.get('connection') as any; + const isConnectionString = + typeof connection === 'string' || connection instanceof String; + const autoParse = typeof parseConnectionString !== 'boolean'; + + const shouldParseConnectionString = autoParse + ? isConnectionString + : parseConnectionString && isConnectionString; + + return shouldParseConnectionString + ? parseMysqlConnectionString(connection as string) + : connection; +} + +/** + * Parses a mysql connection string. + * + * e.g. mysql://examplename:somepassword@examplehost:3306/dbname + * @param connectionString The mysql connection string + */ +export function parseMysqlConnectionString( + connectionString: string, +): Knex.MySqlConnectionConfig { + try { + const { + protocol, + username, + password, + port, + hostname, + pathname, + searchParams, + } = new URL(connectionString); + + if (protocol !== 'mysql:') { + throw new Error(`Unknown protocol ${protocol}`); + } else if (!username || !password) { + throw new Error(`Missing username/password`); + } else if (!pathname.match(/^\/[^/]+$/)) { + throw new Error(`Expected single path segment`); + } + + const result: Knex.MySqlConnectionConfig = { + user: username, + password, + host: hostname, + port: Number(port || 3306), + database: decodeURIComponent(pathname.substr(1)), + }; + + const ssl = searchParams.get('ssl'); + if (ssl) { + result.ssl = ssl; + } + + const debug = searchParams.get('debug'); + if (debug) { + result.debug = yn(debug); + } + + return result; + } catch (e) { + throw new InputError( + `Error while parsing MySQL connection string, ${e}`, + e, + ); + } +} + +/** + * Creates the missing mysql database if it does not exist + * + * @param dbConfig The database config + * @param databases The names of the databases to create + */ +export async function ensureMysqlDatabaseExists( + dbConfig: Config, + ...databases: Array +) { + const admin = createMysqlDatabaseClient(dbConfig, { + connection: { + database: (null as unknown) as string, + }, + }); + + try { + const ensureDatabase = async (database: string) => { + await admin.raw(`CREATE DATABASE IF NOT EXISTS ??`, [database]); + }; + await Promise.all(databases.map(ensureDatabase)); + } finally { + await admin.destroy(); + } +} diff --git a/yarn.lock b/yarn.lock index 23c8118e7e..a21c6ef0f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11495,6 +11495,11 @@ delegates@^1.0.0: resolved "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +denque@^1.4.1: + version "1.5.0" + resolved "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de" + integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ== + depd@^1.1.2, depd@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -13822,6 +13827,13 @@ gcs-resumable-upload@^3.1.4: pumpify "^2.0.0" stream-events "^1.0.4" +generate-function@^2.3.1: + version "2.3.1" + resolved "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz#f069617690c10c868e73b8465746764f97c3479f" + integrity sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ== + dependencies: + is-property "^1.0.2" + generic-names@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872" @@ -15920,6 +15932,11 @@ is-promise@^2.1.0, is-promise@^2.2.2: resolved "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ== +is-property@^1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" + integrity sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ= + is-reference@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7" @@ -17996,7 +18013,7 @@ lru-cache@2: resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952" integrity sha1-bUUk6LlV+V1PW1iFHOId1y+06VI= -lru-cache@^4.0.1: +lru-cache@^4.0.1, lru-cache@^4.1.3: version "4.1.5" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd" integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g== @@ -18668,7 +18685,6 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: resolved "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a" integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ== dependencies: - encoding "^0.1.12" minipass "^3.1.0" minipass-sized "^1.0.3" minizlib "^2.0.0" @@ -18993,6 +19009,20 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" +mysql2@^2.2.5: + version "2.2.5" + resolved "https://registry.npmjs.org/mysql2/-/mysql2-2.2.5.tgz#72624ffb4816f80f96b9c97fedd8c00935f9f340" + integrity sha512-XRqPNxcZTpmFdXbJqb+/CtYVLCx14x1RTeNMD4954L331APu75IC74GDqnZMEt1kwaXy6TySo55rF2F3YJS78g== + dependencies: + denque "^1.4.1" + generate-function "^2.3.1" + iconv-lite "^0.6.2" + long "^4.0.0" + lru-cache "^6.0.0" + named-placeholders "^1.1.2" + seq-queue "^0.0.5" + sqlstring "^2.3.2" + mz@^2.7.0: version "2.7.0" resolved "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32" @@ -19002,6 +19032,13 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +named-placeholders@^1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz#ceb1fbff50b6b33492b5cf214ccf5e39cef3d0e8" + integrity sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA== + dependencies: + lru-cache "^4.1.3" + nan@2.14.1, nan@^2.12.1, nan@^2.14.0: version "2.14.1" resolved "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01" @@ -23532,6 +23569,11 @@ sentence-case@^3.0.4: tslib "^2.0.3" upper-case-first "^2.0.2" +seq-queue@^0.0.5: + version "0.0.5" + resolved "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" + integrity sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4= + serialize-error@^8.0.1, serialize-error@^8.1.0: version "8.1.0" resolved "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz#3a069970c712f78634942ddd50fbbc0eaebe2f67" @@ -24137,6 +24179,11 @@ sqlite3@^5.0.0: optionalDependencies: node-gyp "3.x" +sqlstring@^2.3.2: + version "2.3.2" + resolved "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz#cdae7169389a1375b18e885f2e60b3e460809514" + integrity sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg== + ssh2-streams@~0.4.10: version "0.4.10" resolved "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34"