From cfc3ca6ce060315b461a96a0aababf2a6590c207 Mon Sep 17 00:00:00 2001
From: Greg Cobb <gcobb@vmware.com>
Date: Thu, 8 Jun 2023 15:31:26 -0700
Subject: [PATCH 01/20] patch: Add Continued MySQL Support

Expanded packages/backend-common with MySQL Tests
Updated packages/backend-tasks column types to conform to MySQL
Updated packages/backend-tasks database tests with MySQL
Updated packages/backend-tasks datetime column to work with MySQL
Updated packages/create-app with a production MySQL App Config template
Updated packages/e2e-test to allow for e2e testing with MySQL
Updated plugins/app-backend some text columns to string
Updated plugins/app-backend interval to work with MySQL
Updated plugins/bazaar-backend to run db tests against MySQL
Updated plugins/catalog-backend-module-incremental-ingestion to run
db tests against MySQL
Updated plugins/catalog-backend text columns to longtext to work
with MySQL like issue suggested
Updated plugins/code-coverage-backend text column to string
Updated plugins/linguist-backend text column to string
Updated plugins/tech-insights-backend text columns to string
Updated plugins/tech-insights-backend db tests to include MySQL

Added New E2E tests to run on pull requests to test against MySQL

Co-authored-by: Alex Rocha <alexr1@vmware.com>
Co-authored-by: David Alvarado <dalvarado@vmware.com>
Co-authored-by: Shwetha Gururaj <gururajsh@vmware.com>
Co-authored-by: Al <aberezovsky@vmware.com>
Co-authored-by: Gerg <gcobb@vmware.com>
Signed-off-by: Pete Levine A <lpete@vmware.com>
Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md           | 32 +++++++++
 .github/workflows/test-db.yml                 | 26 ++++++++
 .github/workflows/verify_e2e-linux-mysql.yml  | 66 +++++++++++++++++++
 .../02-testing.md                             |  2 +-
 packages/backend-common/package.json          |  1 +
 .../src/database/DatabaseManager.test.ts      | 27 ++++++++
 .../migrations/20210928160613_init.js         |  2 +-
 packages/backend-tasks/src/migrations.test.ts |  2 +-
 .../src/tasks/PluginTaskSchedulerImpl.test.ts |  2 +-
 .../tasks/PluginTaskSchedulerJanitor.test.ts  |  1 +
 .../src/tasks/TaskScheduler.test.ts           |  2 +-
 .../src/tasks/TaskWorker.test.ts              |  2 +-
 .../backend-tasks/src/tasks/TaskWorker.ts     | 27 +++++---
 packages/backend/package.json                 |  1 +
 .../app-config.production.mysql.yaml          | 29 ++++++++
 packages/e2e-test/package.json                |  1 +
 packages/e2e-test/src/commands/run.ts         | 62 ++++++++++++-----
 .../migrations/20211229105307_init.js         |  2 +-
 .../src/lib/assets/StaticAssetsStore.test.ts  |  8 ++-
 .../src/lib/assets/StaticAssetsStore.ts       |  5 +-
 .../src/service/DatabaseHandler.test.ts       |  2 +-
 .../src/module/WrapperProviders.test.ts       |  2 +-
 .../20210302150147_refresh_state.js           |  4 +-
 .../src/database/DefaultProcessingDatabase.ts |  1 +
 .../migrations/20210302_init.js               |  2 +-
 .../migrations/20221115_init.js               |  2 +-
 .../migrations/202109061111_fact_schemas.js   |  2 +-
 .../migrations/202109061212_facts.js          |  4 +-
 .../service/fact/FactRetrieverEngine.test.ts  |  2 +-
 yarn.lock                                     | 33 ++++++++++
 30 files changed, 307 insertions(+), 47 deletions(-)
 create mode 100644 .changeset/yellow-trains-protect.md
 create mode 100644 .github/workflows/test-db.yml
 create mode 100644 .github/workflows/verify_e2e-linux-mysql.yml
 create mode 100644 packages/create-app/templates/default-app/app-config.production.mysql.yaml

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
new file mode 100644
index 0000000000..1ac73feecf
--- /dev/null
+++ b/.changeset/yellow-trains-protect.md
@@ -0,0 +1,32 @@
+---
+'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch
+'@backstage/plugin-code-coverage-backend': patch
+'@backstage/plugin-tech-insights-backend': patch
+'@backstage/plugin-linguist-backend': patch
+'@backstage/backend-common': patch
+'@backstage/plugin-catalog-backend': patch
+'@backstage/backend-tasks': patch
+'@backstage/plugin-bazaar-backend': patch
+'@backstage/plugin-app-backend': patch
+'e2e-test': patch
+---
+
+Expanded packages/backend-common added test to database manager specifically for MySQL for test that targets postgresql
+Updated packages/backend-tasks text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated packages/backend-tasks database migrations, scheduler and task manager test database array to include MySQL in the tests
+Updated packages/backend-tasks task worker to add the includeOffset flag when updating the startAt column
+Updated packages/create-app with a production MySQL App Config template. This config is referenced in the new e2e Linux MySQL test
+Updated plugins/app-backend text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated plugins/app-backend StaticAssetStore interval column to use the date_sub function to work with MySQL
+Updated plugins/bazaar-backend Database Handler tests to run db tests against MySQL
+Updated plugins/catalog-backend-module-incremental-ingestion WrapperProviders tests database array to run tests against MySQL
+db tests against MySQL
+Updated plugins/catalog-backend text columns to longtext to work
+with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
+Updated plugins/code-coverage-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/linguist-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/tech-insights-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
+Updated plugins/tech-insights-backend FactRetriever engine database list to run against MySQL
+Updated e2e-test run command to run automated e2e tests against a MySQL service
diff --git a/.github/workflows/test-db.yml b/.github/workflows/test-db.yml
new file mode 100644
index 0000000000..89822efffd
--- /dev/null
+++ b/.github/workflows/test-db.yml
@@ -0,0 +1,26 @@
+name: "!Test Databases"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "mysql" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: yarn
+    - run: yarn install
+    - run: yarn tsc
+    - run: yarn build:all
+    - run: yarn test
diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml
new file mode 100644
index 0000000000..caf7c2b3ef
--- /dev/null
+++ b/.github/workflows/verify_e2e-linux-mysql.yml
@@ -0,0 +1,66 @@
+name: E2E Linux MySQL
+on:
+  # NOTE: If you change these you must update verify_e2e-linux-noop.yml as well
+  pull_request:
+    paths-ignore:
+      - '.changeset/**'
+      - 'contrib/**'
+      - 'docs/**'
+      - 'microsite/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      mysql8:
+        image: mysql:8
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: >-
+          --health-cmd "mysqladmin ping -h localhost"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 3306/tcp
+
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version: [16.x]
+
+    env:
+      CI: true
+      NODE_OPTIONS: --max-old-space-size=4096
+
+    name: E2E Linux mysql ${{ matrix.node-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure Git
+        run: |
+          git config --global user.email noreply@backstage.io
+          git config --global user.name 'GitHub e2e user'
+      - name: use node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          registry-url: https://registry.npmjs.org/ # Needed for auth
+          cache: ''
+      - name: yarn install
+        uses: backstage/actions/yarn-install@v0.6.4
+        with:
+          cache-prefix: ${{ runner.os }}-v${{ matrix.node-version }}-${{ github.ref }}
+
+      - run: yarn tsc
+      - run: yarn backstage-cli repo build
+      - name: run E2E test
+        run: |
+          yarn e2e-test run
+        env:
+          BACKSTAGE_TEST_DISABLE_DOCKER: 1
+          BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
diff --git a/docs/backend-system/building-plugins-and-modules/02-testing.md b/docs/backend-system/building-plugins-and-modules/02-testing.md
index fdd8cdb1e3..8ede8855e4 100644
--- a/docs/backend-system/building-plugins-and-modules/02-testing.md
+++ b/docs/backend-system/building-plugins-and-modules/02-testing.md
@@ -118,7 +118,7 @@ describe('MyDatabaseClass', () => {
   // "physical" databases to test against is much costlier than creating the
   // "logical" databases within them that the individual tests use.
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   // Just an example of how to conveniently bundle up the setup code
diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json
index fd9b9cb0df..dee9ea2333 100644
--- a/packages/backend-common/package.json
+++ b/packages/backend-common/package.json
@@ -92,6 +92,7 @@
     "minimatch": "^5.0.0",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
+    "mysql2": "^2.2.5",
     "node-fetch": "^2.6.7",
     "node-forge": "^1.3.1",
     "pg": "^8.3.0",
diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts
index 69386d77c5..bf4af17d05 100644
--- a/packages/backend-common/src/database/DatabaseManager.test.ts
+++ b/packages/backend-common/src/database/DatabaseManager.test.ts
@@ -425,6 +425,33 @@ describe('DatabaseManager', () => {
       );
     });
 
+    it('generates a database name override when prefix is not explicitly set for mysql', async () => {
+      const testManager = DatabaseManager.fromConfig(
+        new ConfigReader({
+          backend: {
+            database: {
+              client: 'mysql',
+              connection: {
+                host: 'localhost',
+                user: 'foo',
+                password: 'bar',
+                database: 'foodb',
+              },
+            },
+          },
+        }),
+      );
+
+      await testManager.forPlugin('testplugin').getClient();
+      const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
+      const [_baseConfig, overrides] = mockCalls[0];
+
+      expect(overrides).toHaveProperty(
+        'connection.database',
+        expect.stringContaining('backstage_plugin_'),
+      );
+    });
+
     it('uses values from plugin connection string if top level client should be used', async () => {
       const pluginId = 'stringoverride';
       await manager.forPlugin(pluginId).getClient();
diff --git a/packages/backend-tasks/migrations/20210928160613_init.js b/packages/backend-tasks/migrations/20210928160613_init.js
index 21d80ae24c..f9900cab11 100644
--- a/packages/backend-tasks/migrations/20210928160613_init.js
+++ b/packages/backend-tasks/migrations/20210928160613_init.js
@@ -26,7 +26,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
     table.comment('Tasks used for scheduling work on multiple workers');
     table
-      .text('id')
+      .string('id')
       .primary()
       .notNullable()
       .comment('The unique ID of this particular task');
diff --git a/packages/backend-tasks/src/migrations.test.ts b/packages/backend-tasks/src/migrations.test.ts
index 7e471a1c16..1d701c971d 100644
--- a/packages/backend-tasks/src/migrations.test.ts
+++ b/packages/backend-tasks/src/migrations.test.ts
@@ -43,7 +43,7 @@ jest.setTimeout(60_000);
 
 describe('migrations', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'MYSQL_8', 'SQLITE_3'],
   });
 
   it.each(databases.eachSupportedId())(
diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
index 6247755bbd..bb6a66e1c2 100644
--- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
+++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
@@ -36,7 +36,7 @@ jest.setTimeout(60_000);
 
 describe('PluginTaskManagerImpl', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   beforeAll(async () => {
diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts
index 5b5d592b7b..efbad2e8b6 100644
--- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts
+++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts
@@ -42,6 +42,7 @@ describe('PluginTaskSchedulerJanitor', () => {
       'POSTGRES_13',
       'POSTGRES_9',
       'SQLITE_3',
+      'MYSQL_8',
     ],
   });
 
diff --git a/packages/backend-tasks/src/tasks/TaskScheduler.test.ts b/packages/backend-tasks/src/tasks/TaskScheduler.test.ts
index 89eff433db..6aa77dd7fe 100644
--- a/packages/backend-tasks/src/tasks/TaskScheduler.test.ts
+++ b/packages/backend-tasks/src/tasks/TaskScheduler.test.ts
@@ -25,7 +25,7 @@ jest.setTimeout(60_000);
 describe('TaskScheduler', () => {
   const logger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   async function createDatabase(
diff --git a/packages/backend-tasks/src/tasks/TaskWorker.test.ts b/packages/backend-tasks/src/tasks/TaskWorker.test.ts
index fa2577ce53..b8b24e045a 100644
--- a/packages/backend-tasks/src/tasks/TaskWorker.test.ts
+++ b/packages/backend-tasks/src/tasks/TaskWorker.test.ts
@@ -28,7 +28,7 @@ jest.setTimeout(60_000);
 describe('TaskWorker', () => {
   const logger = getVoidLogger();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   beforeEach(() => {
diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts
index 483fb1a02d..a03d1ec058 100644
--- a/packages/backend-tasks/src/tasks/TaskWorker.ts
+++ b/packages/backend-tasks/src/tasks/TaskWorker.ts
@@ -175,11 +175,15 @@ export class TaskWorker {
       const time = new CronTime(settings.cadence)
         .sendAt()
         .minus({ seconds: 1 }) // immediately, if "* * * * * *"
-        .toUTC()
-        .toISO();
-      startAt = this.knex.client.config.client.includes('sqlite3')
-        ? this.knex.raw('datetime(?)', [time])
-        : this.knex.raw(`?`, [time]);
+        .toUTC();
+
+      if (this.knex.client.config.client.includes('sqlite3')) {
+        startAt = this.knex.raw('datetime(?)', [time.toISO()]);
+      } else if (this.knex.client.config.client.includes('mysql')) {
+        startAt = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
+      } else {
+        startAt = this.knex.raw(`?`, [time.toISO()]);
+      }
     } else {
       startAt = this.knex.fn.now();
     }
@@ -279,11 +283,16 @@ export class TaskWorker {
 
     let nextRun: Knex.Raw;
     if (isCron) {
-      const time = new CronTime(settings.cadence).sendAt().toUTC().toISO();
+      const time = new CronTime(settings.cadence).sendAt().toUTC();
       this.logger.debug(`task: ${this.taskId} will next occur around ${time}`);
-      nextRun = this.knex.client.config.client.includes('sqlite3')
-        ? this.knex.raw('datetime(?)', [time])
-        : this.knex.raw(`?`, [time]);
+
+      if (this.knex.client.config.client.includes('sqlite3')) {
+        nextRun = this.knex.raw('datetime(?)', [time.toISO()]);
+      } else if (this.knex.client.config.client.includes('mysql')) {
+        nextRun = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
+      } else {
+        nextRun = this.knex.raw(`?`, [time.toISO()]);
+      }
     } else {
       const dt = Duration.fromISO(settings.cadence).as('seconds');
       this.logger.debug(
diff --git a/packages/backend/package.json b/packages/backend/package.json
index 9e0896233d..98fa88f169 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -87,6 +87,7 @@
     "express-prom-bundle": "^6.3.6",
     "express-promise-router": "^4.1.0",
     "luxon": "^3.0.0",
+    "mysql2": "^2.2.5",
     "pg": "^8.3.0",
     "pg-connection-string": "^2.3.0",
     "prom-client": "^14.0.1",
diff --git a/packages/create-app/templates/default-app/app-config.production.mysql.yaml b/packages/create-app/templates/default-app/app-config.production.mysql.yaml
new file mode 100644
index 0000000000..a2d1c2487d
--- /dev/null
+++ b/packages/create-app/templates/default-app/app-config.production.mysql.yaml
@@ -0,0 +1,29 @@
+app:
+  # Should be the same as backend.baseUrl when using the `app-backend` plugin.
+  baseUrl: http://localhost:7007
+
+backend:
+  # Note that the baseUrl should be the URL that the browser and other clients
+  # should use when communicating with the backend, i.e. it needs to be
+  # reachable not just from within the backend host, but from all of your
+  # callers. When its value is "http://localhost:7007", it's strictly private
+  # and can't be reached by others.
+  baseUrl: http://localhost:7007
+  # The listener can also be expressed as a single <host>:<port> string. In this case we bind to
+  # all interfaces, the most permissive setting. The right value depends on your specific deployment.
+  listen: ':7007'
+
+  # config options including ssl: https://github.com/mysqljs/mysql#connection-options
+  database:
+    client: mysql2
+    connection:
+      host: ${MYSQL_HOST}
+      port: ${MYSQL_PORT}
+      user: ${MYSQL_USER}
+      password: ${MYSQL_PASSWORD}
+
+catalog:
+  # Overrides the default list locations from app-config.yaml as these contain example data.
+  # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details
+  # on how to get entities into the catalog.
+  locations: []
diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json
index 48d4643394..50d15c970d 100644
--- a/packages/e2e-test/package.json
+++ b/packages/e2e-test/package.json
@@ -36,6 +36,7 @@
     "fs-extra": "10.1.0",
     "handlebars": "^4.7.3",
     "pgtools": "^1.0.0",
+    "mysql2": "^2.2.5",
     "puppeteer": "^17.0.0",
     "tree-kill": "^1.2.2"
   },
diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index b3fb8d9a3c..e48665fda5 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -31,7 +31,10 @@ import {
   waitForExit,
   print,
 } from '../lib/helpers';
+
+import mysql from 'mysql2/promise';
 import pgtools from 'pgtools';
+
 import { findPaths } from '@backstage/cli-common';
 
 // eslint-disable-next-line no-restricted-syntax
@@ -66,11 +69,17 @@ export async function run() {
   print('Starting the app');
   await testAppServe(pluginId, appDir);
 
-  if (Boolean(process.env.POSTGRES_USER)) {
-    print('Testing the PostgreSQL backend startup');
-    await preCleanPostgres();
+  if (Boolean(process.env.POSTGRES_USER) || Boolean(process.env.MYSQL_USER)) {
+    print('Testing the datbase backend startup');
+    await preCleanDatabase();
     const appConfig = path.resolve(appDir, 'app-config.yaml');
-    const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
+    let productionConfig = path.resolve(appDir, 'app-config.production.yaml');
+    if (Boolean(process.env.MYSQL_HOST)) {
+      productionConfig = path.resolve(
+        appDir,
+        'app-config.production.mysql.yaml',
+      );
+    }
     await testBackendStart(
       appDir,
       '--config',
@@ -79,7 +88,7 @@ export async function run() {
       productionConfig,
     );
   }
-  print('Testing the SQLite backend startup');
+  print('Testing the Database backend startup');
   await testBackendStart(appDir);
 
   if (process.env.CI) {
@@ -427,24 +436,42 @@ async function testAppServe(pluginId: string, appDir: string) {
 }
 
 /** Drops PG databases */
-async function dropDB(database: string) {
-  const config = {
-    host: process.env.POSTGRES_HOST,
-    port: process.env.POSTGRES_PORT,
-    user: process.env.POSTGRES_USER,
-    password: process.env.POSTGRES_PASSWORD,
-  };
-
+async function dropDB(database: string, client: string) {
   try {
-    await pgtools.dropdb(config, database);
+    if (client === 'postgres') {
+      const config = {
+        host: process.env.POSTGRES_HOST ? process.env.POSTGRES_HOST : '',
+        port: process.env.POSTGRES_PORT ? process.env.POSTGRES_PORT : '',
+        user: process.env.POSTGRES_USER ? process.env.POSTGRES_USER : '',
+        password: process.env.POSTGRES_PASSWORD
+          ? process.env.POSTGRES_PASSWORD
+          : '',
+      };
+      await pgtools.dropdb(config, database);
+    } else if (client === 'mysql') {
+      const connectionString =
+        process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING ?? '';
+      const connection = await mysql.createConnection(connectionString);
+      await connection.execute('DROP DATABASE ?', [database]);
+    }
   } catch (_) {
-    /* do nothing*/
+    /* do nothing */
   }
 }
 
 /** Clean remnants from prior e2e runs */
-async function preCleanPostgres() {
+async function preCleanDatabase() {
   print('Dropping old DBs');
+  if (Boolean(process.env.POSTGRES_HOST)) {
+    await dropClientDatabases('postgres');
+  }
+  if (Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)) {
+    await dropClientDatabases('mysql');
+  }
+  print('Created DBs');
+}
+
+async function dropClientDatabases(client: string) {
   await Promise.all(
     [
       'catalog',
@@ -454,9 +481,8 @@ async function preCleanPostgres() {
       'proxy',
       'techdocs',
       'search',
-    ].map(name => dropDB(`backstage_plugin_${name}`)),
+    ].map(name => dropDB(`backstage_plugin_${name}`, client)),
   );
-  print('Created DBs');
 }
 
 /**
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 29e37b35f7..33fccd336d 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,7 +24,7 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table.text('path').primary().notNullable().comment('The path of the file');
+    table.string('path').primary().notNullable().comment('The path of the file');
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())
diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
index 8a0935d4e3..21e86562c6 100644
--- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
+++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
@@ -37,7 +37,7 @@ jest.setTimeout(60_000);
 
 describe('StaticAssetsStore', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   it.each(databases.eachSupportedId())(
@@ -160,7 +160,11 @@ describe('StaticAssetsStore', () => {
         .update({
           last_modified_at: knex.client.config.client.includes('sqlite3')
             ? knex.raw(`datetime('now', '-3600 seconds')`)
-            : knex.raw(`now() + interval '-3600 seconds'`),
+            : (
+              knex.client.config.client.includes('mysql') 
+                ? knex.raw(`date_sub(now(), interval 3600 second)`) 
+                : knex.raw(`now() + interval '-3600 seconds'`)
+            ),
         });
       expect(updated).toBe(1);
 
diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
index 6905d63779..25606575cd 100644
--- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
+++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
@@ -144,7 +144,10 @@ export class StaticAssetsStore implements StaticAssetProvider {
         '<=',
         this.#db.client.config.client.includes('sqlite3')
           ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`])
-          : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`),
+          : (this.#db.client.config.client.includes('mysql') 
+            ? this.#db.raw(`date_sub(now(), interval ${maxAgeSeconds} second)`)
+            : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`)
+          ),
       )
       .delete();
   }
diff --git a/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts b/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts
index 0c78dbdd87..4e46581240 100644
--- a/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts
+++ b/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts
@@ -36,7 +36,7 @@ jest.setTimeout(60_000);
 
 describe('DatabaseHandler', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   function createDatabaseManager(
diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts
index 0d910f664d..1f318c2590 100644
--- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts
+++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts
@@ -24,7 +24,7 @@ import { WrapperProviders } from './WrapperProviders';
 describe('WrapperProviders', () => {
   const applyDatabaseMigrations = jest.fn();
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
   const config = new ConfigReader({});
   const logger = getVoidLogger();
diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
index 1c8081c0ef..f600667727 100644
--- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
+++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
@@ -37,7 +37,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('The unprocessed entity (in original form) as JSON');
     table
-      .text('processed_entity')
+      .text('processed_entity', 'longtext')
       .nullable()
       .comment('The processed entity (not yet stitched) as JSON');
     table
@@ -83,7 +83,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('Random value representing a unique stitch attempt ticket');
     table
-      .text('final_entity')
+      .text('final_entity', 'longtext')
       .nullable()
       .comment('The JSON encoded final entity');
     table.index('entity_id', 'final_entities_entity_id_idx');
diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts
index 4959a33eef..61a062bc4a 100644
--- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts
+++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts
@@ -138,6 +138,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         type,
       }),
     );
+
     await tx.batchInsert(
       'relations',
       this.deduplicateRelations(relationRows),
diff --git a/plugins/code-coverage-backend/migrations/20210302_init.js b/plugins/code-coverage-backend/migrations/20210302_init.js
index a794b6a60c..35eb91541b 100644
--- a/plugins/code-coverage-backend/migrations/20210302_init.js
+++ b/plugins/code-coverage-backend/migrations/20210302_init.js
@@ -34,7 +34,7 @@ exports.up = async function up(knex) {
       .comment('An insert counter to ensure ordering');
     table.uuid('id').notNullable().comment('The ID of the code coverage');
     table
-      .text('entity')
+      .string('entity')
       .notNullable()
       .comment('The entity ref that this code coverage applies to');
     table
diff --git a/plugins/linguist-backend/migrations/20221115_init.js b/plugins/linguist-backend/migrations/20221115_init.js
index 3bcf99e08b..359f7c8359 100644
--- a/plugins/linguist-backend/migrations/20221115_init.js
+++ b/plugins/linguist-backend/migrations/20221115_init.js
@@ -32,7 +32,7 @@ exports.up = async function up(knex) {
       .comment('An insert counter to ensure ordering');
     table.uuid('id').notNullable().comment('The ID of the Linguist result');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .unique()
       .notNullable()
       .comment('The entity ref that this Linguist result applies to');
diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js
index 6094daa4b7..2521f6d8bb 100644
--- a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js
+++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js
@@ -25,7 +25,7 @@ exports.up = async function up(knex) {
       'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.',
     );
     table
-      .text('id')
+      .string('id')
       .notNullable()
       .comment('Identifier of the fact retriever plugin/package');
     table
diff --git a/plugins/tech-insights-backend/migrations/202109061212_facts.js b/plugins/tech-insights-backend/migrations/202109061212_facts.js
index 0bdcdc0da1..237d826869 100644
--- a/plugins/tech-insights-backend/migrations/202109061212_facts.js
+++ b/plugins/tech-insights-backend/migrations/202109061212_facts.js
@@ -25,7 +25,7 @@ exports.up = async function up(knex) {
       'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.',
     );
     table
-      .text('id')
+      .string('id')
       .notNullable()
       .comment('Unique identifier of the fact retriever plugin/package');
     table
@@ -40,7 +40,7 @@ exports.up = async function up(knex) {
       .notNullable()
       .comment('The timestamp when this entry was created');
     table
-      .text('entity')
+      .string('entity')
       .notNullable()
       .comment('Identifier of the entity these facts relate to');
     table
diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
index 5bf05cda98..00b6672d7b 100644
--- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
+++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
@@ -118,7 +118,7 @@ describe('FactRetrieverEngine', () => {
   }
 
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
   });
 
   async function createEngine(
diff --git a/yarn.lock b/yarn.lock
index ec0379375a..7dd2088620 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -23672,6 +23672,7 @@ __metadata:
     handlebars: ^4.7.3
     nodemon: ^3.0.1
     pgtools: ^1.0.0
+    mysql2: ^2.2.5
     puppeteer: ^17.0.0
     tree-kill: ^1.2.2
     ts-node: ^10.0.0
@@ -25272,6 +25273,7 @@ __metadata:
     express-prom-bundle: ^6.3.6
     express-promise-router: ^4.1.0
     luxon: ^3.0.0
+    mysql2: ^2.2.5
     pg: ^8.3.0
     pg-connection-string: ^2.3.0
     prom-client: ^14.0.1
@@ -34812,6 +34814,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pg-connection-string@npm:^2.6.2":
+  version: 2.6.2
+  resolution: "pg-connection-string@npm:2.6.2"
+  checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5
+  languageName: node
+  linkType: hard
+
 "pg-int8@npm:1.0.1":
   version: 1.0.1
   resolution: "pg-int8@npm:1.0.1"
@@ -34894,6 +34903,30 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pg@npm:^8.4.0":
+  version: 8.11.2
+  resolution: "pg@npm:8.11.2"
+  dependencies:
+    buffer-writer: 2.0.0
+    packet-reader: 1.0.0
+    pg-cloudflare: ^1.1.1
+    pg-connection-string: ^2.6.2
+    pg-pool: ^3.6.1
+    pg-protocol: ^1.6.0
+    pg-types: ^2.1.0
+    pgpass: 1.x
+  peerDependencies:
+    pg-native: ">=3.0.1"
+  dependenciesMeta:
+    pg-cloudflare:
+      optional: true
+  peerDependenciesMeta:
+    pg-native:
+      optional: true
+  checksum: dfea8a2269d500dee8c17291207e5a25897163480037beb7a59be35f51e33b519c297c943ea6898b285d6a74a0d661901dc9cff2e587cc4be0bbf09b833a71a5
+  languageName: node
+  linkType: hard
+
 "pgpass@npm:1.x":
   version: 1.0.2
   resolution: "pgpass@npm:1.0.2"

From 957273367a0d42e15ca3ccbb06976d017c7729ec Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 3 Aug 2023 00:22:44 -0400
Subject: [PATCH 02/20] Updated changeset contribution for create-app

Signed-off-by: Pete Levine A <lpete@vmware.com>
Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index 1ac73feecf..f079a23055 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -8,7 +8,7 @@
 '@backstage/backend-tasks': patch
 '@backstage/plugin-bazaar-backend': patch
 '@backstage/plugin-app-backend': patch
-'e2e-test': patch
+'@backstage/create-app': patch
 ---
 
 Expanded packages/backend-common added test to database manager specifically for MySQL for test that targets postgresql

From f651154d34e4837eecdbc357f93563fbb2e0b7eb Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 3 Aug 2023 00:25:17 -0400
Subject: [PATCH 03/20] Remove packages/plugins not referenced in PR

Signed-off-by: Pete Levine A <lpete@vmware.com>
Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index f079a23055..594ae23f3d 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -1,12 +1,10 @@
 ---
-'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch
 '@backstage/plugin-code-coverage-backend': patch
 '@backstage/plugin-tech-insights-backend': patch
 '@backstage/plugin-linguist-backend': patch
 '@backstage/backend-common': patch
 '@backstage/plugin-catalog-backend': patch
 '@backstage/backend-tasks': patch
-'@backstage/plugin-bazaar-backend': patch
 '@backstage/plugin-app-backend': patch
 '@backstage/create-app': patch
 ---

From e43a81811ef63e79814e4a28fc6d652c6638a7cb Mon Sep 17 00:00:00 2001
From: Alex <xandro.rocha@gmail.com>
Date: Mon, 7 Aug 2023 08:32:27 -0500
Subject: [PATCH 04/20] Simplifying changeset message to minimal relevant
 changes across updated plugins (#1)

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index 594ae23f3d..c8abc5429c 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -9,22 +9,4 @@
 '@backstage/create-app': patch
 ---
 
-Expanded packages/backend-common added test to database manager specifically for MySQL for test that targets postgresql
-Updated packages/backend-tasks text columns to longtext to work
-with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
-Updated packages/backend-tasks database migrations, scheduler and task manager test database array to include MySQL in the tests
-Updated packages/backend-tasks task worker to add the includeOffset flag when updating the startAt column
-Updated packages/create-app with a production MySQL App Config template. This config is referenced in the new e2e Linux MySQL test
-Updated plugins/app-backend text columns to longtext to work
-with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
-Updated plugins/app-backend StaticAssetStore interval column to use the date_sub function to work with MySQL
-Updated plugins/bazaar-backend Database Handler tests to run db tests against MySQL
-Updated plugins/catalog-backend-module-incremental-ingestion WrapperProviders tests database array to run tests against MySQL
-db tests against MySQL
-Updated plugins/catalog-backend text columns to longtext to work
-with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration
-Updated plugins/code-coverage-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
-Updated plugins/linguist-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
-Updated plugins/tech-insights-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters
-Updated plugins/tech-insights-backend FactRetriever engine database list to run against MySQL
-Updated e2e-test run command to run automated e2e tests against a MySQL service
+Changes needed to support MySQL

From 7143e55652e6d2d5b1b3b52f94e34ebd3f2ee95f Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Mon, 7 Aug 2023 12:33:05 -0400
Subject: [PATCH 05/20] remove duplicate packages from yarn lock

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 yarn.lock | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 7dd2088620..685ca3b540 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -34814,13 +34814,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg-connection-string@npm:^2.6.2":
-  version: 2.6.2
-  resolution: "pg-connection-string@npm:2.6.2"
-  checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5
-  languageName: node
-  linkType: hard
-
 "pg-int8@npm:1.0.1":
   version: 1.0.1
   resolution: "pg-int8@npm:1.0.1"
@@ -34903,30 +34896,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg@npm:^8.4.0":
-  version: 8.11.2
-  resolution: "pg@npm:8.11.2"
-  dependencies:
-    buffer-writer: 2.0.0
-    packet-reader: 1.0.0
-    pg-cloudflare: ^1.1.1
-    pg-connection-string: ^2.6.2
-    pg-pool: ^3.6.1
-    pg-protocol: ^1.6.0
-    pg-types: ^2.1.0
-    pgpass: 1.x
-  peerDependencies:
-    pg-native: ">=3.0.1"
-  dependenciesMeta:
-    pg-cloudflare:
-      optional: true
-  peerDependenciesMeta:
-    pg-native:
-      optional: true
-  checksum: dfea8a2269d500dee8c17291207e5a25897163480037beb7a59be35f51e33b519c297c943ea6898b285d6a74a0d661901dc9cff2e587cc4be0bbf09b833a71a5
-  languageName: node
-  linkType: hard
-
 "pgpass@npm:1.x":
   version: 1.0.2
   resolution: "pgpass@npm:1.0.2"

From 02cedae992a07a98ff188e45af1d97acc3b5b675 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Mon, 7 Aug 2023 13:56:31 -0400
Subject: [PATCH 06/20] fix linting and prettier issues

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .github/workflows/test-db.yml                 | 25 +++++++++----------
 .../migrations/20211229105307_init.js         |  6 ++++-
 .../src/lib/assets/StaticAssetsStore.test.ts  | 16 ++++++------
 .../src/lib/assets/StaticAssetsStore.ts       | 13 ++++++----
 4 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/test-db.yml b/.github/workflows/test-db.yml
index 89822efffd..515eb64620 100644
--- a/.github/workflows/test-db.yml
+++ b/.github/workflows/test-db.yml
@@ -1,26 +1,25 @@
-name: "!Test Databases"
+name: '!Test Databases'
 
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ "mysql" ]
+    branches: ['mysql']
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [18.x]
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: yarn
-    - run: yarn install
-    - run: yarn tsc
-    - run: yarn build:all
-    - run: yarn test
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: yarn
+      - run: yarn install
+      - run: yarn tsc
+      - run: yarn build:all
+      - run: yarn test
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 33fccd336d..859fc99a14 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,7 +24,11 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table.string('path').primary().notNullable().comment('The path of the file');
+    table
+      .string('path')
+      .primary()
+      .notNullable()
+      .comment('The path of the file');
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())
diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
index 21e86562c6..68fe4e2287 100644
--- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
+++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts
@@ -153,18 +153,18 @@ describe('StaticAssetsStore', () => {
           content: async () => Buffer.alloc(0),
         },
       ]);
-
+      // interval check for postgresql
+      let hourPast = `now() + interval '-3600 seconds'`;
+      if (knex.client.config.client.includes('mysql')) {
+        hourPast = `date_sub(now(), interval 3600 second)`;
+      } else if (knex.client.config.client.includes('sqlite3')) {
+        hourPast = `datetime('now', '-3600 seconds')`;
+      }
       // Rewrite modified time of "old" to be 1h in the past
       const updated = await knex('static_assets_cache')
         .where({ path: 'old' })
         .update({
-          last_modified_at: knex.client.config.client.includes('sqlite3')
-            ? knex.raw(`datetime('now', '-3600 seconds')`)
-            : (
-              knex.client.config.client.includes('mysql') 
-                ? knex.raw(`date_sub(now(), interval 3600 second)`) 
-                : knex.raw(`now() + interval '-3600 seconds'`)
-            ),
+          last_modified_at: knex.raw(hourPast),
         });
       expect(updated).toBe(1);
 
diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
index 25606575cd..c2cfc471fb 100644
--- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
+++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
@@ -138,16 +138,19 @@ export class StaticAssetsStore implements StaticAssetProvider {
    */
   async trimAssets(options: { maxAgeSeconds: number }) {
     const { maxAgeSeconds } = options;
+    let lastModifiedInterval = `now() + interval '${-maxAgeSeconds} seconds'`;
+    if (this.#db.client.config.client.includes('mysql')) {
+      lastModifiedInterval = `date_sub(now(), interval ${maxAgeSeconds} second)`;
+    } else if (this.#db.client.config.client.includes('sqlite3')) {
+      lastModifiedInterval = `datetime('now', ?)`;
+    }
     await this.#db<StaticAssetRow>('static_assets_cache')
       .where(
         'last_modified_at',
         '<=',
         this.#db.client.config.client.includes('sqlite3')
-          ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`])
-          : (this.#db.client.config.client.includes('mysql') 
-            ? this.#db.raw(`date_sub(now(), interval ${maxAgeSeconds} second)`)
-            : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`)
-          ),
+          ? this.#db.raw(lastModifiedInterval, [`-${maxAgeSeconds} seconds`])
+          : this.#db.raw(lastModifiedInterval),
       )
       .delete();
   }

From 6e379187fa4b220f9541f38ff1aee1292d50e525 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Wed, 9 Aug 2023 23:40:39 -0400
Subject: [PATCH 07/20] fix: update migrations used in tests causing error

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .../backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts   | 2 +-
 plugins/bazaar-backend/migrations/20211014144054_init.js      | 4 ++--
 .../migrations/20211117092217_optional_entity_ref.js          | 2 ++
 .../src/service/fact/FactRetrieverEngine.test.ts              | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
index bb6a66e1c2..6247755bbd 100644
--- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
+++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
@@ -36,7 +36,7 @@ jest.setTimeout(60_000);
 
 describe('PluginTaskManagerImpl', () => {
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   beforeAll(async () => {
diff --git a/plugins/bazaar-backend/migrations/20211014144054_init.js b/plugins/bazaar-backend/migrations/20211014144054_init.js
index 503812225d..480771acb4 100644
--- a/plugins/bazaar-backend/migrations/20211014144054_init.js
+++ b/plugins/bazaar-backend/migrations/20211014144054_init.js
@@ -23,7 +23,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('metadata', table => {
     table.comment('The table of Bazaar metadata');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .notNullable()
       .unique()
       .comment('The ref of the entity');
@@ -49,7 +49,7 @@ exports.up = async function up(knex) {
   await knex.schema.createTable('members', table => {
     table.comment('The table of Bazaar members');
     table
-      .text('entity_ref')
+      .string('entity_ref')
       .notNullable()
       .references('metadata.entity_ref')
       .onDelete('CASCADE')
diff --git a/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js b/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js
index 0953d5c2b1..ae2618fbef 100644
--- a/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js
+++ b/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js
@@ -89,9 +89,11 @@ exports.up = async function up(knex) {
     await knex.schema.alterTable('members', table => {
       table
         .integer('item_id')
+        .unsigned()
         .references('metadata.id')
         .onDelete('CASCADE')
         .comment('Id of the associated item');
+      table.dropForeign('entity_ref');
       table.dropColumn('entity_ref');
     });
   }
diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
index 00b6672d7b..5bf05cda98 100644
--- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
+++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
@@ -118,7 +118,7 @@ describe('FactRetrieverEngine', () => {
   }
 
   const databases = TestDatabases.create({
-    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
   });
 
   async function createEngine(

From 8f61efaea576937a5b4608fb147e019658f8ae49 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 10 Aug 2023 09:59:11 -0400
Subject: [PATCH 08/20] chore: update changeset to reflect new fixes

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index c8abc5429c..ceec0edd72 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -7,6 +7,7 @@
 '@backstage/backend-tasks': patch
 '@backstage/plugin-app-backend': patch
 '@backstage/create-app': patch
+'@backstage/plugin-bazaar-backend': patch
 ---
 
 Changes needed to support MySQL

From a2960b0667dbfa43e5bb15a758dc313a5e9f31c3 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Tue, 15 Aug 2023 10:09:05 -0400
Subject: [PATCH 09/20] remove unnecessary testing, fix columns, feedback

Signed-off-by: lpete@vmware.com <lpete@vmware.com>

fix issues with merge

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .github/workflows/test-db.yml                 | 25 -----------
 .github/workflows/verify_e2e-linux-mysql.yml  |  2 +-
 .../app-config.production.mysql.yaml          | 29 -------------
 packages/e2e-test/package.json                |  2 +-
 packages/e2e-test/src/commands/index.ts       |  9 +++-
 packages/e2e-test/src/commands/run.ts         | 41 +++++++------------
 .../migrations/20211229105307_init.js         |  2 +-
 .../src/lib/assets/StaticAssetsStore.ts       | 20 ++++-----
 yarn.lock                                     |  2 +-
 9 files changed, 37 insertions(+), 95 deletions(-)
 delete mode 100644 .github/workflows/test-db.yml
 delete mode 100644 packages/create-app/templates/default-app/app-config.production.mysql.yaml

diff --git a/.github/workflows/test-db.yml b/.github/workflows/test-db.yml
deleted file mode 100644
index 515eb64620..0000000000
--- a/.github/workflows/test-db.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-name: '!Test Databases'
-
-on:
-  workflow_dispatch:
-  pull_request:
-    branches: ['mysql']
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [18.x]
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: yarn
-      - run: yarn install
-      - run: yarn tsc
-      - run: yarn build:all
-      - run: yarn test
diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml
index caf7c2b3ef..3ab6673c8c 100644
--- a/.github/workflows/verify_e2e-linux-mysql.yml
+++ b/.github/workflows/verify_e2e-linux-mysql.yml
@@ -60,7 +60,7 @@ jobs:
       - run: yarn backstage-cli repo build
       - name: run E2E test
         run: |
-          yarn e2e-test run
+          yarn e2e-test run --dbms=mysql
         env:
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
           BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
diff --git a/packages/create-app/templates/default-app/app-config.production.mysql.yaml b/packages/create-app/templates/default-app/app-config.production.mysql.yaml
deleted file mode 100644
index a2d1c2487d..0000000000
--- a/packages/create-app/templates/default-app/app-config.production.mysql.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-app:
-  # Should be the same as backend.baseUrl when using the `app-backend` plugin.
-  baseUrl: http://localhost:7007
-
-backend:
-  # Note that the baseUrl should be the URL that the browser and other clients
-  # should use when communicating with the backend, i.e. it needs to be
-  # reachable not just from within the backend host, but from all of your
-  # callers. When its value is "http://localhost:7007", it's strictly private
-  # and can't be reached by others.
-  baseUrl: http://localhost:7007
-  # The listener can also be expressed as a single <host>:<port> string. In this case we bind to
-  # all interfaces, the most permissive setting. The right value depends on your specific deployment.
-  listen: ':7007'
-
-  # config options including ssl: https://github.com/mysqljs/mysql#connection-options
-  database:
-    client: mysql2
-    connection:
-      host: ${MYSQL_HOST}
-      port: ${MYSQL_PORT}
-      user: ${MYSQL_USER}
-      password: ${MYSQL_PASSWORD}
-
-catalog:
-  # Overrides the default list locations from app-config.yaml as these contain example data.
-  # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details
-  # on how to get entities into the catalog.
-  locations: []
diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json
index 50d15c970d..7e6ebdd749 100644
--- a/packages/e2e-test/package.json
+++ b/packages/e2e-test/package.json
@@ -35,8 +35,8 @@
     "cross-fetch": "^3.1.5",
     "fs-extra": "10.1.0",
     "handlebars": "^4.7.3",
-    "pgtools": "^1.0.0",
     "mysql2": "^2.2.5",
+    "pgtools": "^1.0.0",
     "puppeteer": "^17.0.0",
     "tree-kill": "^1.2.2"
   },
diff --git a/packages/e2e-test/src/commands/index.ts b/packages/e2e-test/src/commands/index.ts
index 95d311427c..c5740a1d6f 100644
--- a/packages/e2e-test/src/commands/index.ts
+++ b/packages/e2e-test/src/commands/index.ts
@@ -18,5 +18,12 @@ import { Command } from 'commander';
 import { run } from './run';
 
 export function registerCommands(program: Command) {
-  program.command('run').description('Run e2e tests').action(run);
+  program
+    .command('run')
+    .description('Run e2e tests')
+    .option(
+      '-db, --dbms',
+      'Selected database management system for e2e testing, (eg. mysql, postgres)',
+    )
+    .action(run);
 }
diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index e48665fda5..0ea542f213 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -46,7 +46,7 @@ const templatePackagePaths = [
   'packages/create-app/templates/default-app/packages/backend/package.json.hbs',
 ];
 
-export async function run() {
+export async function run(dbms: string = 'postgres') {
   const rootDir = await fs.mkdtemp(resolvePath(os.tmpdir(), 'backstage-e2e-'));
   print(`CLI E2E test root: ${rootDir}\n`);
 
@@ -69,17 +69,15 @@ export async function run() {
   print('Starting the app');
   await testAppServe(pluginId, appDir);
 
-  if (Boolean(process.env.POSTGRES_USER) || Boolean(process.env.MYSQL_USER)) {
+  if (
+    Boolean(process.env.POSTGRES_USER) ||
+    Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)
+  ) {
     print('Testing the datbase backend startup');
-    await preCleanDatabase();
+    await dropClientDatabases(dbms);
     const appConfig = path.resolve(appDir, 'app-config.yaml');
-    let productionConfig = path.resolve(appDir, 'app-config.production.yaml');
-    if (Boolean(process.env.MYSQL_HOST)) {
-      productionConfig = path.resolve(
-        appDir,
-        'app-config.production.mysql.yaml',
-      );
-    }
+    const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
+
     await testBackendStart(
       appDir,
       '--config',
@@ -435,10 +433,10 @@ async function testAppServe(pluginId: string, appDir: string) {
   }
 }
 
-/** Drops PG databases */
-async function dropDB(database: string, client: string) {
+/** Drops databases */
+async function dropDB(database: string, dbms: string) {
   try {
-    if (client === 'postgres') {
+    if (dbms === 'postgres') {
       const config = {
         host: process.env.POSTGRES_HOST ? process.env.POSTGRES_HOST : '',
         port: process.env.POSTGRES_PORT ? process.env.POSTGRES_PORT : '',
@@ -448,7 +446,7 @@ async function dropDB(database: string, client: string) {
           : '',
       };
       await pgtools.dropdb(config, database);
-    } else if (client === 'mysql') {
+    } else if (dbms === 'mysql') {
       const connectionString =
         process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING ?? '';
       const connection = await mysql.createConnection(connectionString);
@@ -460,18 +458,8 @@ async function dropDB(database: string, client: string) {
 }
 
 /** Clean remnants from prior e2e runs */
-async function preCleanDatabase() {
+async function dropClientDatabases(dbms: string) {
   print('Dropping old DBs');
-  if (Boolean(process.env.POSTGRES_HOST)) {
-    await dropClientDatabases('postgres');
-  }
-  if (Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)) {
-    await dropClientDatabases('mysql');
-  }
-  print('Created DBs');
-}
-
-async function dropClientDatabases(client: string) {
   await Promise.all(
     [
       'catalog',
@@ -481,8 +469,9 @@ async function dropClientDatabases(client: string) {
       'proxy',
       'techdocs',
       'search',
-    ].map(name => dropDB(`backstage_plugin_${name}`, client)),
+    ].map(name => dropDB(`backstage_plugin_${name}`, dbms)),
   );
+  print('Created DBs');
 }
 
 /**
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 859fc99a14..5f1bfdc30e 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -25,7 +25,7 @@ exports.up = async function up(knex) {
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
     table
-      .string('path')
+      .text('path', 'longtext')
       .primary()
       .notNullable()
       .comment('The path of the file');
diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
index c2cfc471fb..9f6ada717d 100644
--- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
+++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts
@@ -138,20 +138,20 @@ export class StaticAssetsStore implements StaticAssetProvider {
    */
   async trimAssets(options: { maxAgeSeconds: number }) {
     const { maxAgeSeconds } = options;
-    let lastModifiedInterval = `now() + interval '${-maxAgeSeconds} seconds'`;
+    let lastModifiedInterval = this.#db.raw(
+      `now() + interval '${-maxAgeSeconds} seconds'`,
+    );
     if (this.#db.client.config.client.includes('mysql')) {
-      lastModifiedInterval = `date_sub(now(), interval ${maxAgeSeconds} second)`;
+      lastModifiedInterval = this.#db.raw(
+        `date_sub(now(), interval ${maxAgeSeconds} second)`,
+      );
     } else if (this.#db.client.config.client.includes('sqlite3')) {
-      lastModifiedInterval = `datetime('now', ?)`;
+      lastModifiedInterval = this.#db.raw(`datetime('now', ?)`, [
+        `-${maxAgeSeconds} seconds`,
+      ]);
     }
     await this.#db<StaticAssetRow>('static_assets_cache')
-      .where(
-        'last_modified_at',
-        '<=',
-        this.#db.client.config.client.includes('sqlite3')
-          ? this.#db.raw(lastModifiedInterval, [`-${maxAgeSeconds} seconds`])
-          : this.#db.raw(lastModifiedInterval),
-      )
+      .where('last_modified_at', '<=', lastModifiedInterval)
       .delete();
   }
 }
diff --git a/yarn.lock b/yarn.lock
index 685ca3b540..cb8197c964 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -23670,9 +23670,9 @@ __metadata:
     cross-fetch: ^3.1.5
     fs-extra: 10.1.0
     handlebars: ^4.7.3
+    mysql2: ^2.2.5
     nodemon: ^3.0.1
     pgtools: ^1.0.0
-    mysql2: ^2.2.5
     puppeteer: ^17.0.0
     tree-kill: ^1.2.2
     ts-node: ^10.0.0

From a0e6241fc055fef28b1c2cfb0ecb2ad476acaf99 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Tue, 15 Aug 2023 10:49:16 -0400
Subject: [PATCH 10/20] fix argument for e2e-test run

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .github/workflows/verify_e2e-linux-mysql.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml
index 3ab6673c8c..a0f1cae48d 100644
--- a/.github/workflows/verify_e2e-linux-mysql.yml
+++ b/.github/workflows/verify_e2e-linux-mysql.yml
@@ -60,7 +60,7 @@ jobs:
       - run: yarn backstage-cli repo build
       - name: run E2E test
         run: |
-          yarn e2e-test run --dbms=mysql
+          yarn e2e-test run --dbms mysql
         env:
           BACKSTAGE_TEST_DISABLE_DOCKER: 1
           BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored

From 274a321ceaa17f825a64452effa33302c07d0f8d Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Tue, 15 Aug 2023 10:52:20 -0400
Subject: [PATCH 11/20] no longer patching create-app

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index ceec0edd72..62ff99f202 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -6,7 +6,6 @@
 '@backstage/plugin-catalog-backend': patch
 '@backstage/backend-tasks': patch
 '@backstage/plugin-app-backend': patch
-'@backstage/create-app': patch
 '@backstage/plugin-bazaar-backend': patch
 ---
 

From ba7bdec12634bdaf7f6eb8f21752f1e2b4d865e2 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Tue, 15 Aug 2023 16:07:28 -0400
Subject: [PATCH 12/20] 64KB length for mysql support, dbms option for e2e

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 packages/e2e-test/cli-report.md                       | 3 ++-
 packages/e2e-test/src/commands/index.ts               | 3 ++-
 packages/e2e-test/src/commands/run.ts                 | 4 ++--
 plugins/app-backend/migrations/20211229105307_init.js | 3 ++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/packages/e2e-test/cli-report.md b/packages/e2e-test/cli-report.md
index 3a3912b8fc..c79a831bec 100644
--- a/packages/e2e-test/cli-report.md
+++ b/packages/e2e-test/cli-report.md
@@ -12,7 +12,7 @@ Options:
   -h, --help
 
 Commands:
-  run
+  run [options]
   help [command]
 ```
 
@@ -22,5 +22,6 @@ Commands:
 Usage: e2e-test run [options]
 
 Options:
+  -db, --dbms <value>
   -h, --help
 ```
diff --git a/packages/e2e-test/src/commands/index.ts b/packages/e2e-test/src/commands/index.ts
index c5740a1d6f..d2770d6a1e 100644
--- a/packages/e2e-test/src/commands/index.ts
+++ b/packages/e2e-test/src/commands/index.ts
@@ -22,8 +22,9 @@ export function registerCommands(program: Command) {
     .command('run')
     .description('Run e2e tests')
     .option(
-      '-db, --dbms',
+      '-db, --dbms <value>',
       'Selected database management system for e2e testing, (eg. mysql, postgres)',
+      'postgres',
     )
     .action(run);
 }
diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index 0ea542f213..46be27c188 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -46,7 +46,7 @@ const templatePackagePaths = [
   'packages/create-app/templates/default-app/packages/backend/package.json.hbs',
 ];
 
-export async function run(dbms: string = 'postgres') {
+export async function run(options: { dbms: string }) {
   const rootDir = await fs.mkdtemp(resolvePath(os.tmpdir(), 'backstage-e2e-'));
   print(`CLI E2E test root: ${rootDir}\n`);
 
@@ -74,7 +74,7 @@ export async function run(dbms: string = 'postgres') {
     Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)
   ) {
     print('Testing the datbase backend startup');
-    await dropClientDatabases(dbms);
+    await dropClientDatabases(options.dbms);
     const appConfig = path.resolve(appDir, 'app-config.yaml');
     const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
 
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 5f1bfdc30e..4121272bd7 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,8 +24,9 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
+    // setting to 64KB to account for long paths
     table
-      .text('path', 'longtext')
+      .string('path', 65535)
       .primary()
       .notNullable()
       .comment('The path of the file');

From 6ec403f3f0ef8698100bf2a05e9b26165535407f Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Wed, 16 Aug 2023 14:54:57 -0400
Subject: [PATCH 13/20] Path length to maximum pk, rm e2e mysql for now

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .github/workflows/verify_e2e-linux-mysql.yml  | 66 -------------------
 .../migrations/20211229105307_init.js         |  2 +-
 2 files changed, 1 insertion(+), 67 deletions(-)
 delete mode 100644 .github/workflows/verify_e2e-linux-mysql.yml

diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml
deleted file mode 100644
index a0f1cae48d..0000000000
--- a/.github/workflows/verify_e2e-linux-mysql.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-name: E2E Linux MySQL
-on:
-  # NOTE: If you change these you must update verify_e2e-linux-noop.yml as well
-  pull_request:
-    paths-ignore:
-      - '.changeset/**'
-      - 'contrib/**'
-      - 'docs/**'
-      - 'microsite/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    services:
-      mysql8:
-        image: mysql:8
-        env:
-          MYSQL_ROOT_PASSWORD: root
-        options: >-
-          --health-cmd "mysqladmin ping -h localhost"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 3306/tcp
-
-    strategy:
-      fail-fast: true
-      matrix:
-        node-version: [16.x]
-
-    env:
-      CI: true
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    name: E2E Linux mysql ${{ matrix.node-version }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Configure Git
-        run: |
-          git config --global user.email noreply@backstage.io
-          git config --global user.name 'GitHub e2e user'
-      - name: use node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          registry-url: https://registry.npmjs.org/ # Needed for auth
-          cache: ''
-      - name: yarn install
-        uses: backstage/actions/yarn-install@v0.6.4
-        with:
-          cache-prefix: ${{ runner.os }}-v${{ matrix.node-version }}-${{ github.ref }}
-
-      - run: yarn tsc
-      - run: yarn backstage-cli repo build
-      - name: run E2E test
-        run: |
-          yarn e2e-test run --dbms mysql
-        env:
-          BACKSTAGE_TEST_DISABLE_DOCKER: 1
-          BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 4121272bd7..eb08f60e6a 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -26,7 +26,7 @@ exports.up = async function up(knex) {
     );
     // setting to 64KB to account for long paths
     table
-      .string('path', 65535)
+      .string('path', 16383)
       .primary()
       .notNullable()
       .comment('The path of the file');

From 9ae110cde357423046c541197885dc141685fad8 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 17 Aug 2023 11:49:52 -0400
Subject: [PATCH 14/20] Swap pk for app-backend path, fix heartbeat task

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .changeset/yellow-trains-protect.md           |  1 +
 .../migrations/20211229105307_init.js         |  6 ++---
 .../src/scaffolder/tasks/DatabaseTaskStore.ts | 23 +++++++++++--------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md
index 62ff99f202..4b8c50ac3e 100644
--- a/.changeset/yellow-trains-protect.md
+++ b/.changeset/yellow-trains-protect.md
@@ -7,6 +7,7 @@
 '@backstage/backend-tasks': patch
 '@backstage/plugin-app-backend': patch
 '@backstage/plugin-bazaar-backend': patch
+'@backstage/plugin-scaffolder-backend': patch
 ---
 
 Changes needed to support MySQL
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index eb08f60e6a..dfc5f1229b 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,12 +24,12 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    // setting to 64KB to account for long paths
     table
-      .string('path', 16383)
+      .uuid('id')
       .primary()
       .notNullable()
-      .comment('The path of the file');
+      .comment('Auto-generated ID of the asset');
+    table.text('path', 'text').notNullable().comment('The path of the file');
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts
index f3cb9b645e..c96bd6b9cd 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts
@@ -261,18 +261,21 @@ export class DatabaseTaskStore implements TaskStore {
     tasks: { taskId: string }[];
   }> {
     const { timeoutS } = options;
-
+    let heartbeatInterval = this.db.raw(`? - interval '${timeoutS} seconds'`, [
+      this.db.fn.now(),
+    ]);
+    if (this.db.client.config.client.includes('mysql')) {
+      heartbeatInterval = this.db.raw(
+        `date_sub(now(), interval ${timeoutS} second)`,
+      );
+    } else if (this.db.client.config.client.includes('sqlite3')) {
+      heartbeatInterval = this.db.raw(`datetime('now', ?)`, [
+        `-${timeoutS} seconds`,
+      ]);
+    }
     const rawRows = await this.db<RawDbTaskRow>('tasks')
       .where('status', 'processing')
-      .andWhere(
-        'last_heartbeat_at',
-        '<=',
-        this.db.client.config.client.includes('sqlite3')
-          ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`])
-          : this.db.raw(`? - interval '${timeoutS} seconds'`, [
-              this.db.fn.now(),
-            ]),
-      );
+      .andWhere('last_heartbeat_at', '<=', heartbeatInterval);
     const tasks = rawRows.map(row => ({
       taskId: row.id,
     }));

From 2a6250bb4ea41a034a17fcc4f3626b22815038c3 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 17 Aug 2023 12:47:53 -0400
Subject: [PATCH 15/20] fix pk for path for the stat assets, fix run e2e

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 packages/e2e-test/src/commands/run.ts         | 35 +++++++++++--------
 .../migrations/20211229105307_init.js         |  7 +---
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index 46be27c188..3254feab98 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -46,7 +46,7 @@ const templatePackagePaths = [
   'packages/create-app/templates/default-app/packages/backend/package.json.hbs',
 ];
 
-export async function run(options: { dbms: string }) {
+export async function run() {
   const rootDir = await fs.mkdtemp(resolvePath(os.tmpdir(), 'backstage-e2e-'));
   print(`CLI E2E test root: ${rootDir}\n`);
 
@@ -69,15 +69,11 @@ export async function run(options: { dbms: string }) {
   print('Starting the app');
   await testAppServe(pluginId, appDir);
 
-  if (
-    Boolean(process.env.POSTGRES_USER) ||
-    Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)
-  ) {
-    print('Testing the datbase backend startup');
-    await dropClientDatabases(options.dbms);
+  if (Boolean(process.env.POSTGRES_USER) || Boolean(process.env.MYSQL_USER)) {
+    print('Testing the database backend startup');
+    await preCleanDatabase();
     const appConfig = path.resolve(appDir, 'app-config.yaml');
     const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
-
     await testBackendStart(
       appDir,
       '--config',
@@ -433,10 +429,10 @@ async function testAppServe(pluginId: string, appDir: string) {
   }
 }
 
-/** Drops databases */
-async function dropDB(database: string, dbms: string) {
+/** Drops PG databases */
+async function dropDB(database: string, client: string) {
   try {
-    if (dbms === 'postgres') {
+    if (client === 'postgres') {
       const config = {
         host: process.env.POSTGRES_HOST ? process.env.POSTGRES_HOST : '',
         port: process.env.POSTGRES_PORT ? process.env.POSTGRES_PORT : '',
@@ -446,7 +442,7 @@ async function dropDB(database: string, dbms: string) {
           : '',
       };
       await pgtools.dropdb(config, database);
-    } else if (dbms === 'mysql') {
+    } else if (client === 'mysql') {
       const connectionString =
         process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING ?? '';
       const connection = await mysql.createConnection(connectionString);
@@ -458,8 +454,18 @@ async function dropDB(database: string, dbms: string) {
 }
 
 /** Clean remnants from prior e2e runs */
-async function dropClientDatabases(dbms: string) {
+async function preCleanDatabase() {
   print('Dropping old DBs');
+  if (Boolean(process.env.POSTGRES_HOST)) {
+    await dropClientDatabases('postgres');
+  }
+  if (Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)) {
+    await dropClientDatabases('mysql');
+  }
+  print('Created DBs');
+}
+
+async function dropClientDatabases(client: string) {
   await Promise.all(
     [
       'catalog',
@@ -469,9 +475,8 @@ async function dropClientDatabases(dbms: string) {
       'proxy',
       'techdocs',
       'search',
-    ].map(name => dropDB(`backstage_plugin_${name}`, dbms)),
+    ].map(name => dropDB(`backstage_plugin_${name}`, client)),
   );
-  print('Created DBs');
 }
 
 /**
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index dfc5f1229b..29e37b35f7 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,12 +24,7 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table
-      .uuid('id')
-      .primary()
-      .notNullable()
-      .comment('Auto-generated ID of the asset');
-    table.text('path', 'text').notNullable().comment('The path of the file');
+    table.text('path').primary().notNullable().comment('The path of the file');
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())

From ca061f5a9d9b403d3e794f3d44aad50a784cece9 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 17 Aug 2023 14:52:30 -0400
Subject: [PATCH 16/20] replace pk with unique due to size constraints

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 plugins/app-backend/migrations/20211229105307_init.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 29e37b35f7..5367d5af60 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -24,7 +24,11 @@ exports.up = async function up(knex) {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table.text('path').primary().notNullable().comment('The path of the file');
+    table
+      .text('path', 'text')
+      .unique()
+      .notNullable()
+      .comment('The path of the file');
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())

From bfa604de333f3f93eb1483cea2c33886b8b6f473 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Thu, 17 Aug 2023 17:38:24 -0400
Subject: [PATCH 17/20] remove pk for mysql db

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 .../migrations/20211229105307_init.js            | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 5367d5af60..9a05f97198 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -20,15 +20,21 @@
  * @param {import('knex').Knex} knex
  */
 exports.up = async function up(knex) {
+  const isMySQL = knex.client.config.client.includes('mysql');
   return knex.schema.createTable('static_assets_cache', table => {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
-    table
-      .text('path', 'text')
-      .unique()
-      .notNullable()
-      .comment('The path of the file');
+    if (!isMySQL) {
+      table
+        .text('path')
+        .primary()
+        .notNullable()
+        .comment('The path of the file');
+    } else {
+      // cannot have a unique key in mysql with a text column greater than 733 bytes
+      table.text('path').notNullable().comment('The path of the file');
+    }
     table
       .dateTime('last_modified_at')
       .defaultTo(knex.fn.now())

From 90c19f86cee105a6a3c74fa4741dca341e9f8e0c Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Fri, 18 Aug 2023 15:42:24 -0400
Subject: [PATCH 18/20] remove unnecessary dbms string, fix ternary

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 packages/e2e-test/src/commands/index.ts | 10 +---------
 packages/e2e-test/src/commands/run.ts   | 17 +++++++----------
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/packages/e2e-test/src/commands/index.ts b/packages/e2e-test/src/commands/index.ts
index d2770d6a1e..95d311427c 100644
--- a/packages/e2e-test/src/commands/index.ts
+++ b/packages/e2e-test/src/commands/index.ts
@@ -18,13 +18,5 @@ import { Command } from 'commander';
 import { run } from './run';
 
 export function registerCommands(program: Command) {
-  program
-    .command('run')
-    .description('Run e2e tests')
-    .option(
-      '-db, --dbms <value>',
-      'Selected database management system for e2e testing, (eg. mysql, postgres)',
-      'postgres',
-    )
-    .action(run);
+  program.command('run').description('Run e2e tests').action(run);
 }
diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index 3254feab98..b57e426116 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -434,17 +434,14 @@ async function dropDB(database: string, client: string) {
   try {
     if (client === 'postgres') {
       const config = {
-        host: process.env.POSTGRES_HOST ? process.env.POSTGRES_HOST : '',
-        port: process.env.POSTGRES_PORT ? process.env.POSTGRES_PORT : '',
-        user: process.env.POSTGRES_USER ? process.env.POSTGRES_USER : '',
-        password: process.env.POSTGRES_PASSWORD
-          ? process.env.POSTGRES_PASSWORD
-          : '',
+        host: process.env.POSTGRES_HOST,
+        port: process.env.POSTGRES_PORT,
+        user: process.env.POSTGRES_USER,
+        password: process.env.POSTGRES_PASSWORD,
       };
       await pgtools.dropdb(config, database);
     } else if (client === 'mysql') {
-      const connectionString =
-        process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING ?? '';
+      const connectionString = process.env.MYSQL_CONNECTION ?? '';
       const connection = await mysql.createConnection(connectionString);
       await connection.execute('DROP DATABASE ?', [database]);
     }
@@ -459,10 +456,10 @@ async function preCleanDatabase() {
   if (Boolean(process.env.POSTGRES_HOST)) {
     await dropClientDatabases('postgres');
   }
-  if (Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)) {
+  if (Boolean(process.env.MYSQL_CONNECTION)) {
     await dropClientDatabases('mysql');
   }
-  print('Created DBs');
+  print('Dropped DBs');
 }
 
 async function dropClientDatabases(client: string) {

From 3f8558b3f589a4aed912ed63cb0bef5e5291db38 Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Fri, 18 Aug 2023 16:01:28 -0400
Subject: [PATCH 19/20] update api report with updated e2e test run cmd

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 packages/e2e-test/cli-report.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/e2e-test/cli-report.md b/packages/e2e-test/cli-report.md
index c79a831bec..3a3912b8fc 100644
--- a/packages/e2e-test/cli-report.md
+++ b/packages/e2e-test/cli-report.md
@@ -12,7 +12,7 @@ Options:
   -h, --help
 
 Commands:
-  run [options]
+  run
   help [command]
 ```
 
@@ -22,6 +22,5 @@ Commands:
 Usage: e2e-test run [options]
 
 Options:
-  -db, --dbms <value>
   -h, --help
 ```

From e57d7188cdf892cc2c2f8282369aab608f5657be Mon Sep 17 00:00:00 2001
From: "lpete@vmware.com" <lpete@vmware.com>
Date: Mon, 21 Aug 2023 11:02:55 -0400
Subject: [PATCH 20/20] set unique index for mysql, set connection var

Signed-off-by: lpete@vmware.com <lpete@vmware.com>
---
 packages/e2e-test/src/commands/run.ts                 | 5 ++++-
 plugins/app-backend/migrations/20211229105307_init.js | 9 +++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts
index b57e426116..20b2753f6b 100644
--- a/packages/e2e-test/src/commands/run.ts
+++ b/packages/e2e-test/src/commands/run.ts
@@ -69,7 +69,10 @@ export async function run() {
   print('Starting the app');
   await testAppServe(pluginId, appDir);
 
-  if (Boolean(process.env.POSTGRES_USER) || Boolean(process.env.MYSQL_USER)) {
+  if (
+    Boolean(process.env.POSTGRES_USER) ||
+    Boolean(process.env.MYSQL_CONNECTION)
+  ) {
     print('Testing the database backend startup');
     await preCleanDatabase();
     const appConfig = path.resolve(appDir, 'app-config.yaml');
diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js
index 9a05f97198..25ed48c307 100644
--- a/plugins/app-backend/migrations/20211229105307_init.js
+++ b/plugins/app-backend/migrations/20211229105307_init.js
@@ -21,7 +21,7 @@
  */
 exports.up = async function up(knex) {
   const isMySQL = knex.client.config.client.includes('mysql');
-  return knex.schema.createTable('static_assets_cache', table => {
+  await knex.schema.createTable('static_assets_cache', table => {
     table.comment(
       'A cache of static assets that where previously deployed and may still be lazy-loaded by clients',
     );
@@ -32,7 +32,6 @@ exports.up = async function up(knex) {
         .notNullable()
         .comment('The path of the file');
     } else {
-      // cannot have a unique key in mysql with a text column greater than 733 bytes
       table.text('path').notNullable().comment('The path of the file');
     }
     table
@@ -45,6 +44,12 @@ exports.up = async function up(knex) {
     table.binary('content').notNullable().comment('The asset content');
     table.index('last_modified_at', 'static_asset_cache_last_modified_at_idx');
   });
+  // specifically for mysql specify a unique index up to 254 characters(mysql limit)
+  if (isMySQL) {
+    await knex.schema.raw(
+      'create unique index static_assets_cache_path_idx on static_assets_cache(path(254));',
+    );
+  }
 };
 
 /**