Prep work for mysql support in backend-common

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
2021-05-09 22:42:56 +02:00
parent 72ac10b135
commit f9fb4a205e
8 changed files with 416 additions and 7 deletions
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Prep work for mysql support in backend-common
@@ -177,6 +177,7 @@ mkdocs
 monorepo
 monorepos
 msw
+mysql
 namespace
 namespaced
 namespaces
@@ -62,7 +62,8 @@
    "stoppable": "^1.1.0",
    "tar": "^6.0.5",
    "unzipper": "^0.10.11",
-    "winston": "^3.2.1"
+    "winston": "^3.2.1",
+    "yn": "^4.0.0"
  },
  "peerDependencies": {
    "pg-connection-string": "^2.3.0"
@@ -94,6 +95,7 @@
    "jest": "^26.0.1",
    "mock-fs": "^4.13.0",
    "msw": "^0.21.2",
+    "mysql2": "^2.2.5",
    "recursive-readdir": "^2.2.2",
    "supertest": "^6.1.3"
  },
@@ -46,11 +46,11 @@ describe('database connection', () => {
      ).toBeTruthy();
    });

-    it('tries to create a mysql connection as a passthrough', () => {
+    it('returns a mysql connection', () => {
      expect(() =>
        createDatabaseClient(
          new ConfigReader({
-            client: 'mysql',
+            client: 'mysql2',
            connection: {
              host: '127.0.0.1',
              user: 'foo',
@@ -59,7 +59,7 @@ describe('database connection', () => {
            },
          }),
        ),
-      ).toThrowError(/Cannot find module 'mysql'/);
+      ).toBeTruthy();
    });

    it('accepts overrides', () => {
@@ -14,9 +14,10 @@
 * limitations under the License.
 */

-import knexFactory, { Knex } from 'knex';
 import { Config } from '@backstage/config';
+import knexFactory, { Knex } from 'knex';
 import { mergeDatabaseConfig } from './config';
+import { createMysqlDatabaseClient, ensureMysqlDatabaseExists } from './mysql';
 import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres';
 import { createSqliteDatabaseClient } from './sqlite3';

@@ -36,6 +37,8 @@ export function createDatabaseClient(

  if (client === 'pg') {
    return createPgDatabaseClient(dbConfig, overrides);
+  } else if (client === 'mysql' || client === 'mysql2') {
+    return createMysqlDatabaseClient(dbConfig, overrides);
  } else if (client === 'sqlite3') {
    return createSqliteDatabaseClient(dbConfig, overrides);
  }
@@ -60,6 +63,8 @@ export async function ensureDatabaseExists(

  if (client === 'pg') {
    return ensurePgDatabaseExists(dbConfig, ...databases);
+  } else if (client === 'mysql' || client === 'mysql2') {
+    return ensureMysqlDatabaseExists(dbConfig, ...databases);
  }

  return undefined;
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import {
+  buildMysqlDatabaseConfig,
+  createMysqlDatabaseClient,
+  getMysqlConnectionConfig,
+  parseMysqlConnectionString,
+} from './mysql';
+
+describe('mysql', () => {
+  const createMockConnection = () => ({
+    host: 'acme',
+    user: 'foo',
+    password: 'bar',
+    database: 'foodb',
+  });
+
+  const createMockConnectionString = () => 'mysql://foo:bar@acme:3306/foodb';
+
+  const createConfig = (connection: any): Config =>
+    new ConfigReader({ client: 'mysql2', connection });
+
+  describe('buildMysqlDatabaseConfig', () => {
+    it('builds a mysql config', () => {
+      const mockConnection = createMockConnection();
+
+      expect(buildMysqlDatabaseConfig(createConfig(mockConnection))).toEqual({
+        client: 'mysql2',
+        connection: mockConnection,
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a connection string config', () => {
+      const mockConnectionString = createMockConnectionString();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnectionString)),
+      ).toEqual({
+        client: 'mysql2',
+        connection: mockConnectionString,
+        useNullAsDefault: true,
+      });
+    });
+
+    it('overrides the database name', () => {
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnection), {
+          connection: { database: 'other_db' },
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('adds additional config settings', () => {
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnection), {
+          connection: { database: 'other_db' },
+          pool: { min: 0, max: 7 },
+          debug: true,
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+        pool: { min: 0, max: 7 },
+        debug: true,
+      });
+    });
+
+    it('overrides the database from connection string', () => {
+      const mockConnectionString = createMockConnectionString();
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnectionString), {
+          connection: { database: 'other_db' },
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          port: 3306,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+      });
+    });
+  });
+
+  describe('getMysqlConnectionConfig', () => {
+    it('returns the connection object back', () => {
+      const mockConnection = createMockConnection();
+      const config = createConfig(mockConnection);
+
+      expect(getMysqlConnectionConfig(config)).toEqual(mockConnection);
+    });
+
+    it('does not parse the connection string', () => {
+      const mockConnection = createMockConnection();
+      const config = createConfig(mockConnection);
+
+      expect(getMysqlConnectionConfig(config, true)).toEqual(mockConnection);
+    });
+
+    it('automatically parses the connection string', () => {
+      const mockConnection = createMockConnection();
+      const mockConnectionString = createMockConnectionString();
+      const config = createConfig(mockConnectionString);
+
+      expect(getMysqlConnectionConfig(config)).toEqual({
+        ...mockConnection,
+        port: 3306,
+      });
+    });
+
+    it('parses the connection string', () => {
+      const mockConnection = createMockConnection();
+      const mockConnectionString = createMockConnectionString();
+      const config = createConfig(mockConnectionString);
+
+      expect(getMysqlConnectionConfig(config, true)).toEqual({
+        ...mockConnection,
+        port: 3306,
+      });
+    });
+  });
+
+  describe('createMysqlDatabaseClient', () => {
+    it('creates a mysql knex instance', () => {
+      expect(
+        createMysqlDatabaseClient(
+          createConfig({
+            host: 'acme',
+            user: 'foo',
+            password: 'bar',
+            database: 'foodb',
+          }),
+        ),
+      ).toBeTruthy();
+    });
+  });
+
+  describe('parseMysqlConnectionString', () => {
+    it('parses a connection string uri', () => {
+      expect(
+        parseMysqlConnectionString(
+          'mysql://u:pass@foobar:3307/dbname?ssl=require',
+        ),
+      ).toEqual({
+        host: 'foobar',
+        user: 'u',
+        password: 'pass',
+        port: 3307,
+        database: 'dbname',
+        ssl: 'require',
+      });
+    });
+  });
+});
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import { InputError } from '@backstage/errors';
+import knexFactory, { Knex } from 'knex';
+import { mergeDatabaseConfig } from './config';
+import yn from 'yn';
+
+/**
+ * Creates a knex mysql database connection
+ *
+ * @param dbConfig The database config
+ * @param overrides Additional options to merge with the config
+ */
+export function createMysqlDatabaseClient(
+  dbConfig: Config,
+  overrides?: Knex.Config,
+) {
+  const knexConfig = buildMysqlDatabaseConfig(dbConfig, overrides);
+  const database = knexFactory(knexConfig);
+  return database;
+}
+
+/**
+ * Builds a knex mysql database connection
+ *
+ * @param dbConfig The database config
+ * @param overrides Additional options to merge with the config
+ */
+export function buildMysqlDatabaseConfig(
+  dbConfig: Config,
+  overrides?: Knex.Config,
+) {
+  return mergeDatabaseConfig(
+    dbConfig.get(),
+    {
+      connection: getMysqlConnectionConfig(dbConfig, !!overrides),
+      useNullAsDefault: true,
+    },
+    overrides,
+  );
+}
+
+/**
+ * Gets the mysql connection config
+ *
+ * @param dbConfig The database config
+ * @param parseConnectionString Flag to explicitly control connection string parsing
+ */
+export function getMysqlConnectionConfig(
+  dbConfig: Config,
+  parseConnectionString?: boolean,
+): Knex.MySqlConnectionConfig | string {
+  const connection = dbConfig.get('connection') as any;
+  const isConnectionString =
+    typeof connection === 'string' || connection instanceof String;
+  const autoParse = typeof parseConnectionString !== 'boolean';
+
+  const shouldParseConnectionString = autoParse
+    ? isConnectionString
+    : parseConnectionString && isConnectionString;
+
+  return shouldParseConnectionString
+    ? parseMysqlConnectionString(connection as string)
+    : connection;
+}
+
+/**
+ * Parses a mysql connection string.
+ *
+ * e.g. mysql://examplename:somepassword@examplehost:3306/dbname
+ * @param connectionString The mysql connection string
+ */
+export function parseMysqlConnectionString(
+  connectionString: string,
+): Knex.MySqlConnectionConfig {
+  try {
+    const {
+      protocol,
+      username,
+      password,
+      port,
+      hostname,
+      pathname,
+      searchParams,
+    } = new URL(connectionString);
+
+    if (protocol !== 'mysql:') {
+      throw new Error(`Unknown protocol ${protocol}`);
+    } else if (!username || !password) {
+      throw new Error(`Missing username/password`);
+    } else if (!pathname.match(/^\/[^/]+$/)) {
+      throw new Error(`Expected single path segment`);
+    }
+
+    const result: Knex.MySqlConnectionConfig = {
+      user: username,
+      password,
+      host: hostname,
+      port: Number(port || 3306),
+      database: decodeURIComponent(pathname.substr(1)),
+    };
+
+    const ssl = searchParams.get('ssl');
+    if (ssl) {
+      result.ssl = ssl;
+    }
+
+    const debug = searchParams.get('debug');
+    if (debug) {
+      result.debug = yn(debug);
+    }
+
+    return result;
+  } catch (e) {
+    throw new InputError(
+      `Error while parsing MySQL connection string, ${e}`,
+      e,
+    );
+  }
+}
+
+/**
+ * Creates the missing mysql database if it does not exist
+ *
+ * @param dbConfig The database config
+ * @param databases The names of the databases to create
+ */
+export async function ensureMysqlDatabaseExists(
+  dbConfig: Config,
+  ...databases: Array<string>
+) {
+  const admin = createMysqlDatabaseClient(dbConfig, {
+    connection: {
+      database: (null as unknown) as string,
+    },
+  });
+
+  try {
+    const ensureDatabase = async (database: string) => {
+      await admin.raw(`CREATE DATABASE IF NOT EXISTS ??`, [database]);
+    };
+    await Promise.all(databases.map(ensureDatabase));
+  } finally {
+    await admin.destroy();
+  }
+}
@@ -11495,6 +11495,11 @@ delegates@^1.0.0:
  resolved "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
  integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=

+denque@^1.4.1:
+  version "1.5.0"
+  resolved "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
+  integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==
+
 depd@^1.1.2, depd@~1.1.2:
  version "1.1.2"
  resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
@@ -13822,6 +13827,13 @@ gcs-resumable-upload@^3.1.4:
    pumpify "^2.0.0"
    stream-events "^1.0.4"

+generate-function@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz#f069617690c10c868e73b8465746764f97c3479f"
+  integrity sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==
+  dependencies:
+    is-property "^1.0.2"
+
 generic-names@^2.0.1:
  version "2.0.1"
  resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872"
@@ -15920,6 +15932,11 @@ is-promise@^2.1.0, is-promise@^2.2.2:
  resolved "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1"
  integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==

+is-property@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84"
+  integrity sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=
+
 is-reference@^1.2.1:
  version "1.2.1"
  resolved "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7"
@@ -17996,7 +18013,7 @@ lru-cache@2:
  resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
  integrity sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=

-lru-cache@^4.0.1:
+lru-cache@^4.0.1, lru-cache@^4.1.3:
  version "4.1.5"
  resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
  integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==
@@ -18668,7 +18685,6 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2:
  resolved "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a"
  integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ==
  dependencies:
-    encoding "^0.1.12"
    minipass "^3.1.0"
    minipass-sized "^1.0.3"
    minizlib "^2.0.0"
@@ -18993,6 +19009,20 @@ mv@~2:
    ncp "~2.0.0"
    rimraf "~2.4.0"

+mysql2@^2.2.5:
+  version "2.2.5"
+  resolved "https://registry.npmjs.org/mysql2/-/mysql2-2.2.5.tgz#72624ffb4816f80f96b9c97fedd8c00935f9f340"
+  integrity sha512-XRqPNxcZTpmFdXbJqb+/CtYVLCx14x1RTeNMD4954L331APu75IC74GDqnZMEt1kwaXy6TySo55rF2F3YJS78g==
+  dependencies:
+    denque "^1.4.1"
+    generate-function "^2.3.1"
+    iconv-lite "^0.6.2"
+    long "^4.0.0"
+    lru-cache "^6.0.0"
+    named-placeholders "^1.1.2"
+    seq-queue "^0.0.5"
+    sqlstring "^2.3.2"
+
 mz@^2.7.0:
  version "2.7.0"
  resolved "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
@@ -19002,6 +19032,13 @@ mz@^2.7.0:
    object-assign "^4.0.1"
    thenify-all "^1.0.0"

+named-placeholders@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz#ceb1fbff50b6b33492b5cf214ccf5e39cef3d0e8"
+  integrity sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==
+  dependencies:
+    lru-cache "^4.1.3"
+
 nan@2.14.1, nan@^2.12.1, nan@^2.14.0:
  version "2.14.1"
  resolved "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
@@ -23532,6 +23569,11 @@ sentence-case@^3.0.4:
    tslib "^2.0.3"
    upper-case-first "^2.0.2"

+seq-queue@^0.0.5:
+  version "0.0.5"
+  resolved "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
+  integrity sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=
+
 serialize-error@^8.0.1, serialize-error@^8.1.0:
  version "8.1.0"
  resolved "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz#3a069970c712f78634942ddd50fbbc0eaebe2f67"
@@ -24137,6 +24179,11 @@ sqlite3@^5.0.0:
  optionalDependencies:
    node-gyp "3.x"

+sqlstring@^2.3.2:
+  version "2.3.2"
+  resolved "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz#cdae7169389a1375b18e885f2e60b3e460809514"
+  integrity sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg==
+
 ssh2-streams@~0.4.10:
  version "0.4.10"
  resolved "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34"