Merge pull request #5625 from backstage/freben/mysql-prep

Prep work for mysql support in backend-common

packages/backend-common/package.json

@@ -64,7 +64,8 @@
     "stoppable": "^1.1.0",
     "tar": "^6.0.5",
     "unzipper": "^0.10.11",
-    "winston": "^3.2.1"
+    "winston": "^3.2.1",
+    "yn": "^4.0.0"
   },
   "peerDependencies": {
     "pg-connection-string": "^2.3.0"
@@ -96,6 +97,7 @@
     "jest": "^26.0.1",
     "mock-fs": "^4.13.0",
     "msw": "^0.21.2",
+    "mysql2": "^2.2.5",
     "recursive-readdir": "^2.2.2",
     "supertest": "^6.1.3"
   },

packages/backend-common/src/database/connection.test.ts

@@ -46,11 +46,11 @@ describe('database connection', () => {
       ).toBeTruthy();
     });
 
-    it('tries to create a mysql connection as a passthrough', () => {
+    it('returns a mysql connection', () => {
       expect(() =>
         createDatabaseClient(
           new ConfigReader({
-            client: 'mysql',
+            client: 'mysql2',
             connection: {
               host: '127.0.0.1',
               user: 'foo',
@@ -59,7 +59,7 @@ describe('database connection', () => {
             },
           }),
         ),
-      ).toThrowError(/Cannot find module 'mysql'/);
+      ).toBeTruthy();
     });
 
     it('accepts overrides', () => {

packages/backend-common/src/database/connection.ts

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-import knexFactory, { Knex } from 'knex';
 import { Config } from '@backstage/config';
+import knexFactory, { Knex } from 'knex';
 import { mergeDatabaseConfig } from './config';
+import { createMysqlDatabaseClient, ensureMysqlDatabaseExists } from './mysql';
 import { createPgDatabaseClient, ensurePgDatabaseExists } from './postgres';
 import { createSqliteDatabaseClient } from './sqlite3';
 
@@ -36,6 +37,8 @@ export function createDatabaseClient(
 
   if (client === 'pg') {
     return createPgDatabaseClient(dbConfig, overrides);
+  } else if (client === 'mysql' || client === 'mysql2') {
+    return createMysqlDatabaseClient(dbConfig, overrides);
   } else if (client === 'sqlite3') {
     return createSqliteDatabaseClient(dbConfig, overrides);
   }
@@ -60,6 +63,8 @@ export async function ensureDatabaseExists(
 
   if (client === 'pg') {
     return ensurePgDatabaseExists(dbConfig, ...databases);
+  } else if (client === 'mysql' || client === 'mysql2') {
+    return ensureMysqlDatabaseExists(dbConfig, ...databases);
   }
 
   return undefined;

packages/backend-common/src/database/mysql.test.ts

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import {
+  buildMysqlDatabaseConfig,
+  createMysqlDatabaseClient,
+  getMysqlConnectionConfig,
+  parseMysqlConnectionString,
+} from './mysql';
+
+describe('mysql', () => {
+  const createMockConnection = () => ({
+    host: 'acme',
+    user: 'foo',
+    password: 'bar',
+    database: 'foodb',
+  });
+
+  const createMockConnectionString = () => 'mysql://foo:bar@acme:3306/foodb';
+
+  const createConfig = (connection: any): Config =>
+    new ConfigReader({ client: 'mysql2', connection });
+
+  describe('buildMysqlDatabaseConfig', () => {
+    it('builds a mysql config', () => {
+      const mockConnection = createMockConnection();
+
+      expect(buildMysqlDatabaseConfig(createConfig(mockConnection))).toEqual({
+        client: 'mysql2',
+        connection: mockConnection,
+        useNullAsDefault: true,
+      });
+    });
+
+    it('builds a connection string config', () => {
+      const mockConnectionString = createMockConnectionString();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnectionString)),
+      ).toEqual({
+        client: 'mysql2',
+        connection: mockConnectionString,
+        useNullAsDefault: true,
+      });
+    });
+
+    it('overrides the database name', () => {
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnection), {
+          connection: { database: 'other_db' },
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+      });
+    });
+
+    it('adds additional config settings', () => {
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnection), {
+          connection: { database: 'other_db' },
+          pool: { min: 0, max: 7 },
+          debug: true,
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+        pool: { min: 0, max: 7 },
+        debug: true,
+      });
+    });
+
+    it('overrides the database from connection string', () => {
+      const mockConnectionString = createMockConnectionString();
+      const mockConnection = createMockConnection();
+
+      expect(
+        buildMysqlDatabaseConfig(createConfig(mockConnectionString), {
+          connection: { database: 'other_db' },
+        }),
+      ).toEqual({
+        client: 'mysql2',
+        connection: {
+          ...mockConnection,
+          port: 3306,
+          database: 'other_db',
+        },
+        useNullAsDefault: true,
+      });
+    });
+  });
+
+  describe('getMysqlConnectionConfig', () => {
+    it('returns the connection object back', () => {
+      const mockConnection = createMockConnection();
+      const config = createConfig(mockConnection);
+
+      expect(getMysqlConnectionConfig(config)).toEqual(mockConnection);
+    });
+
+    it('does not parse the connection string', () => {
+      const mockConnection = createMockConnection();
+      const config = createConfig(mockConnection);
+
+      expect(getMysqlConnectionConfig(config, true)).toEqual(mockConnection);
+    });
+
+    it('automatically parses the connection string', () => {
+      const mockConnection = createMockConnection();
+      const mockConnectionString = createMockConnectionString();
+      const config = createConfig(mockConnectionString);
+
+      expect(getMysqlConnectionConfig(config)).toEqual({
+        ...mockConnection,
+        port: 3306,
+      });
+    });
+
+    it('parses the connection string', () => {
+      const mockConnection = createMockConnection();
+      const mockConnectionString = createMockConnectionString();
+      const config = createConfig(mockConnectionString);
+
+      expect(getMysqlConnectionConfig(config, true)).toEqual({
+        ...mockConnection,
+        port: 3306,
+      });
+    });
+  });
+
+  describe('createMysqlDatabaseClient', () => {
+    it('creates a mysql knex instance', () => {
+      expect(
+        createMysqlDatabaseClient(
+          createConfig({
+            host: 'acme',
+            user: 'foo',
+            password: 'bar',
+            database: 'foodb',
+          }),
+        ),
+      ).toBeTruthy();
+    });
+  });
+
+  describe('parseMysqlConnectionString', () => {
+    it('parses a connection string uri', () => {
+      expect(
+        parseMysqlConnectionString(
+          'mysql://u:pass@foobar:3307/dbname?ssl=require',
+        ),
+      ).toEqual({
+        host: 'foobar',
+        user: 'u',
+        password: 'pass',
+        port: 3307,
+        database: 'dbname',
+        ssl: 'require',
+      });
+    });
+  });
+});

packages/backend-common/src/database/mysql.ts

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import { InputError } from '@backstage/errors';
+import knexFactory, { Knex } from 'knex';
+import { mergeDatabaseConfig } from './config';
+import yn from 'yn';
+
+/**
+ * Creates a knex mysql database connection
+ *
+ * @param dbConfig The database config
+ * @param overrides Additional options to merge with the config
+ */
+export function createMysqlDatabaseClient(
+  dbConfig: Config,
+  overrides?: Knex.Config,
+) {
+  const knexConfig = buildMysqlDatabaseConfig(dbConfig, overrides);
+  const database = knexFactory(knexConfig);
+  return database;
+}
+
+/**
+ * Builds a knex mysql database connection
+ *
+ * @param dbConfig The database config
+ * @param overrides Additional options to merge with the config
+ */
+export function buildMysqlDatabaseConfig(
+  dbConfig: Config,
+  overrides?: Knex.Config,
+) {
+  return mergeDatabaseConfig(
+    dbConfig.get(),
+    {
+      connection: getMysqlConnectionConfig(dbConfig, !!overrides),
+      useNullAsDefault: true,
+    },
+    overrides,
+  );
+}
+
+/**
+ * Gets the mysql connection config
+ *
+ * @param dbConfig The database config
+ * @param parseConnectionString Flag to explicitly control connection string parsing
+ */
+export function getMysqlConnectionConfig(
+  dbConfig: Config,
+  parseConnectionString?: boolean,
+): Knex.MySqlConnectionConfig | string {
+  const connection = dbConfig.get('connection') as any;
+  const isConnectionString =
+    typeof connection === 'string' || connection instanceof String;
+  const autoParse = typeof parseConnectionString !== 'boolean';
+
+  const shouldParseConnectionString = autoParse
+    ? isConnectionString
+    : parseConnectionString && isConnectionString;
+
+  return shouldParseConnectionString
+    ? parseMysqlConnectionString(connection as string)
+    : connection;
+}
+
+/**
+ * Parses a mysql connection string.
+ *
+ * e.g. mysql://examplename:somepassword@examplehost:3306/dbname
+ * @param connectionString The mysql connection string
+ */
+export function parseMysqlConnectionString(
+  connectionString: string,
+): Knex.MySqlConnectionConfig {
+  try {
+    const {
+      protocol,
+      username,
+      password,
+      port,
+      hostname,
+      pathname,
+      searchParams,
+    } = new URL(connectionString);
+
+    if (protocol !== 'mysql:') {
+      throw new Error(`Unknown protocol ${protocol}`);
+    } else if (!username || !password) {
+      throw new Error(`Missing username/password`);
+    } else if (!pathname.match(/^\/[^/]+$/)) {
+      throw new Error(`Expected single path segment`);
+    }
+
+    const result: Knex.MySqlConnectionConfig = {
+      user: username,
+      password,
+      host: hostname,
+      port: Number(port || 3306),
+      database: decodeURIComponent(pathname.substr(1)),
+    };
+
+    const ssl = searchParams.get('ssl');
+    if (ssl) {
+      result.ssl = ssl;
+    }
+
+    const debug = searchParams.get('debug');
+    if (debug) {
+      result.debug = yn(debug);
+    }
+
+    return result;
+  } catch (e) {
+    throw new InputError(
+      `Error while parsing MySQL connection string, ${e}`,
+      e,
+    );
+  }
+}
+
+/**
+ * Creates the missing mysql database if it does not exist
+ *
+ * @param dbConfig The database config
+ * @param databases The names of the databases to create
+ */
+export async function ensureMysqlDatabaseExists(
+  dbConfig: Config,
+  ...databases: Array<string>
+) {
+  const admin = createMysqlDatabaseClient(dbConfig, {
+    connection: {
+      database: (null as unknown) as string,
+    },
+  });
+
+  try {
+    const ensureDatabase = async (database: string) => {
+      await admin.raw(`CREATE DATABASE IF NOT EXISTS ??`, [database]);
+    };
+    await Promise.all(databases.map(ensureDatabase));
+  } finally {
+    await admin.destroy();
+  }
+}