diff --git a/.changeset/yellow-trains-protect.md b/.changeset/yellow-trains-protect.md new file mode 100644 index 0000000000..1ac73feecf --- /dev/null +++ b/.changeset/yellow-trains-protect.md @@ -0,0 +1,32 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-tech-insights-backend': patch +'@backstage/plugin-linguist-backend': patch +'@backstage/backend-common': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/backend-tasks': patch +'@backstage/plugin-bazaar-backend': patch +'@backstage/plugin-app-backend': patch +'e2e-test': patch +--- + +Expanded packages/backend-common added test to database manager specifically for MySQL for test that targets postgresql +Updated packages/backend-tasks text columns to longtext to work +with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration +Updated packages/backend-tasks database migrations, scheduler and task manager test database array to include MySQL in the tests +Updated packages/backend-tasks task worker to add the includeOffset flag when updating the startAt column +Updated packages/create-app with a production MySQL App Config template. This config is referenced in the new e2e Linux MySQL test +Updated plugins/app-backend text columns to longtext to work +with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration +Updated plugins/app-backend StaticAssetStore interval column to use the date_sub function to work with MySQL +Updated plugins/bazaar-backend Database Handler tests to run db tests against MySQL +Updated plugins/catalog-backend-module-incremental-ingestion WrapperProviders tests database array to run tests against MySQL +db tests against MySQL +Updated plugins/catalog-backend text columns to longtext to work +with MySQL. MySQL requires lengths on their varchar columns. String columns in knex default to 255 characters unless otherwise provided in configuration +Updated plugins/code-coverage-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters +Updated plugins/linguist-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters +Updated plugins/tech-insights-backend Migration sql text columns to string to work with MySQL. MySQL requires lengths on varchar columns. String columns default to 255 characters +Updated plugins/tech-insights-backend FactRetriever engine database list to run against MySQL +Updated e2e-test run command to run automated e2e tests against a MySQL service diff --git a/.github/workflows/test-db.yml b/.github/workflows/test-db.yml new file mode 100644 index 0000000000..89822efffd --- /dev/null +++ b/.github/workflows/test-db.yml @@ -0,0 +1,26 @@ +name: "!Test Databases" + +on: + workflow_dispatch: + pull_request: + branches: [ "mysql" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - run: yarn install + - run: yarn tsc + - run: yarn build:all + - run: yarn test diff --git a/.github/workflows/verify_e2e-linux-mysql.yml b/.github/workflows/verify_e2e-linux-mysql.yml new file mode 100644 index 0000000000..caf7c2b3ef --- /dev/null +++ b/.github/workflows/verify_e2e-linux-mysql.yml @@ -0,0 +1,66 @@ +name: E2E Linux MySQL +on: + # NOTE: If you change these you must update verify_e2e-linux-noop.yml as well + pull_request: + paths-ignore: + - '.changeset/**' + - 'contrib/**' + - 'docs/**' + - 'microsite/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + services: + mysql8: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: root + options: >- + --health-cmd "mysqladmin ping -h localhost" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 3306/tcp + + strategy: + fail-fast: true + matrix: + node-version: [16.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + name: E2E Linux mysql ${{ matrix.node-version }} + steps: + - uses: actions/checkout@v3 + + - name: Configure Git + run: | + git config --global user.email noreply@backstage.io + git config --global user.name 'GitHub e2e user' + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + cache: '' + - name: yarn install + uses: backstage/actions/yarn-install@v0.6.4 + with: + cache-prefix: ${{ runner.os }}-v${{ matrix.node-version }}-${{ github.ref }} + + - run: yarn tsc + - run: yarn backstage-cli repo build + - name: run E2E test + run: | + yarn e2e-test run + env: + BACKSTAGE_TEST_DISABLE_DOCKER: 1 + BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored diff --git a/docs/backend-system/building-plugins-and-modules/02-testing.md b/docs/backend-system/building-plugins-and-modules/02-testing.md index fdd8cdb1e3..8ede8855e4 100644 --- a/docs/backend-system/building-plugins-and-modules/02-testing.md +++ b/docs/backend-system/building-plugins-and-modules/02-testing.md @@ -118,7 +118,7 @@ describe('MyDatabaseClass', () => { // "physical" databases to test against is much costlier than creating the // "logical" databases within them that the individual tests use. const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); // Just an example of how to conveniently bundle up the setup code diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index fd9b9cb0df..dee9ea2333 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -92,6 +92,7 @@ "minimatch": "^5.0.0", "minimist": "^1.2.5", "morgan": "^1.10.0", + "mysql2": "^2.2.5", "node-fetch": "^2.6.7", "node-forge": "^1.3.1", "pg": "^8.3.0", diff --git a/packages/backend-common/src/database/DatabaseManager.test.ts b/packages/backend-common/src/database/DatabaseManager.test.ts index 69386d77c5..bf4af17d05 100644 --- a/packages/backend-common/src/database/DatabaseManager.test.ts +++ b/packages/backend-common/src/database/DatabaseManager.test.ts @@ -425,6 +425,33 @@ describe('DatabaseManager', () => { ); }); + it('generates a database name override when prefix is not explicitly set for mysql', async () => { + const testManager = DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'mysql', + connection: { + host: 'localhost', + user: 'foo', + password: 'bar', + database: 'foodb', + }, + }, + }, + }), + ); + + await testManager.forPlugin('testplugin').getClient(); + const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1); + const [_baseConfig, overrides] = mockCalls[0]; + + expect(overrides).toHaveProperty( + 'connection.database', + expect.stringContaining('backstage_plugin_'), + ); + }); + it('uses values from plugin connection string if top level client should be used', async () => { const pluginId = 'stringoverride'; await manager.forPlugin(pluginId).getClient(); diff --git a/packages/backend-tasks/migrations/20210928160613_init.js b/packages/backend-tasks/migrations/20210928160613_init.js index 21d80ae24c..f9900cab11 100644 --- a/packages/backend-tasks/migrations/20210928160613_init.js +++ b/packages/backend-tasks/migrations/20210928160613_init.js @@ -26,7 +26,7 @@ exports.up = async function up(knex) { await knex.schema.createTable('backstage_backend_tasks__tasks', table => { table.comment('Tasks used for scheduling work on multiple workers'); table - .text('id') + .string('id') .primary() .notNullable() .comment('The unique ID of this particular task'); diff --git a/packages/backend-tasks/src/migrations.test.ts b/packages/backend-tasks/src/migrations.test.ts index 7e471a1c16..1d701c971d 100644 --- a/packages/backend-tasks/src/migrations.test.ts +++ b/packages/backend-tasks/src/migrations.test.ts @@ -43,7 +43,7 @@ jest.setTimeout(60_000); describe('migrations', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'MYSQL_8', 'SQLITE_3'], }); it.each(databases.eachSupportedId())( diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts index 6247755bbd..bb6a66e1c2 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts @@ -36,7 +36,7 @@ jest.setTimeout(60_000); describe('PluginTaskManagerImpl', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); beforeAll(async () => { diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts index 5b5d592b7b..efbad2e8b6 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts @@ -42,6 +42,7 @@ describe('PluginTaskSchedulerJanitor', () => { 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', + 'MYSQL_8', ], }); diff --git a/packages/backend-tasks/src/tasks/TaskScheduler.test.ts b/packages/backend-tasks/src/tasks/TaskScheduler.test.ts index 89eff433db..6aa77dd7fe 100644 --- a/packages/backend-tasks/src/tasks/TaskScheduler.test.ts +++ b/packages/backend-tasks/src/tasks/TaskScheduler.test.ts @@ -25,7 +25,7 @@ jest.setTimeout(60_000); describe('TaskScheduler', () => { const logger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); async function createDatabase( diff --git a/packages/backend-tasks/src/tasks/TaskWorker.test.ts b/packages/backend-tasks/src/tasks/TaskWorker.test.ts index fa2577ce53..b8b24e045a 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.test.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.test.ts @@ -28,7 +28,7 @@ jest.setTimeout(60_000); describe('TaskWorker', () => { const logger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); beforeEach(() => { diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index 483fb1a02d..a03d1ec058 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -175,11 +175,15 @@ export class TaskWorker { const time = new CronTime(settings.cadence) .sendAt() .minus({ seconds: 1 }) // immediately, if "* * * * * *" - .toUTC() - .toISO(); - startAt = this.knex.client.config.client.includes('sqlite3') - ? this.knex.raw('datetime(?)', [time]) - : this.knex.raw(`?`, [time]); + .toUTC(); + + if (this.knex.client.config.client.includes('sqlite3')) { + startAt = this.knex.raw('datetime(?)', [time.toISO()]); + } else if (this.knex.client.config.client.includes('mysql')) { + startAt = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]); + } else { + startAt = this.knex.raw(`?`, [time.toISO()]); + } } else { startAt = this.knex.fn.now(); } @@ -279,11 +283,16 @@ export class TaskWorker { let nextRun: Knex.Raw; if (isCron) { - const time = new CronTime(settings.cadence).sendAt().toUTC().toISO(); + const time = new CronTime(settings.cadence).sendAt().toUTC(); this.logger.debug(`task: ${this.taskId} will next occur around ${time}`); - nextRun = this.knex.client.config.client.includes('sqlite3') - ? this.knex.raw('datetime(?)', [time]) - : this.knex.raw(`?`, [time]); + + if (this.knex.client.config.client.includes('sqlite3')) { + nextRun = this.knex.raw('datetime(?)', [time.toISO()]); + } else if (this.knex.client.config.client.includes('mysql')) { + nextRun = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]); + } else { + nextRun = this.knex.raw(`?`, [time.toISO()]); + } } else { const dt = Duration.fromISO(settings.cadence).as('seconds'); this.logger.debug( diff --git a/packages/backend/package.json b/packages/backend/package.json index 9e0896233d..98fa88f169 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -87,6 +87,7 @@ "express-prom-bundle": "^6.3.6", "express-promise-router": "^4.1.0", "luxon": "^3.0.0", + "mysql2": "^2.2.5", "pg": "^8.3.0", "pg-connection-string": "^2.3.0", "prom-client": "^14.0.1", diff --git a/packages/create-app/templates/default-app/app-config.production.mysql.yaml b/packages/create-app/templates/default-app/app-config.production.mysql.yaml new file mode 100644 index 0000000000..a2d1c2487d --- /dev/null +++ b/packages/create-app/templates/default-app/app-config.production.mysql.yaml @@ -0,0 +1,29 @@ +app: + # Should be the same as backend.baseUrl when using the `app-backend` plugin. + baseUrl: http://localhost:7007 + +backend: + # Note that the baseUrl should be the URL that the browser and other clients + # should use when communicating with the backend, i.e. it needs to be + # reachable not just from within the backend host, but from all of your + # callers. When its value is "http://localhost:7007", it's strictly private + # and can't be reached by others. + baseUrl: http://localhost:7007 + # The listener can also be expressed as a single : string. In this case we bind to + # all interfaces, the most permissive setting. The right value depends on your specific deployment. + listen: ':7007' + + # config options including ssl: https://github.com/mysqljs/mysql#connection-options + database: + client: mysql2 + connection: + host: ${MYSQL_HOST} + port: ${MYSQL_PORT} + user: ${MYSQL_USER} + password: ${MYSQL_PASSWORD} + +catalog: + # Overrides the default list locations from app-config.yaml as these contain example data. + # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details + # on how to get entities into the catalog. + locations: [] diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index 48d4643394..50d15c970d 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -36,6 +36,7 @@ "fs-extra": "10.1.0", "handlebars": "^4.7.3", "pgtools": "^1.0.0", + "mysql2": "^2.2.5", "puppeteer": "^17.0.0", "tree-kill": "^1.2.2" }, diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index b3fb8d9a3c..e48665fda5 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -31,7 +31,10 @@ import { waitForExit, print, } from '../lib/helpers'; + +import mysql from 'mysql2/promise'; import pgtools from 'pgtools'; + import { findPaths } from '@backstage/cli-common'; // eslint-disable-next-line no-restricted-syntax @@ -66,11 +69,17 @@ export async function run() { print('Starting the app'); await testAppServe(pluginId, appDir); - if (Boolean(process.env.POSTGRES_USER)) { - print('Testing the PostgreSQL backend startup'); - await preCleanPostgres(); + if (Boolean(process.env.POSTGRES_USER) || Boolean(process.env.MYSQL_USER)) { + print('Testing the datbase backend startup'); + await preCleanDatabase(); const appConfig = path.resolve(appDir, 'app-config.yaml'); - const productionConfig = path.resolve(appDir, 'app-config.production.yaml'); + let productionConfig = path.resolve(appDir, 'app-config.production.yaml'); + if (Boolean(process.env.MYSQL_HOST)) { + productionConfig = path.resolve( + appDir, + 'app-config.production.mysql.yaml', + ); + } await testBackendStart( appDir, '--config', @@ -79,7 +88,7 @@ export async function run() { productionConfig, ); } - print('Testing the SQLite backend startup'); + print('Testing the Database backend startup'); await testBackendStart(appDir); if (process.env.CI) { @@ -427,24 +436,42 @@ async function testAppServe(pluginId: string, appDir: string) { } /** Drops PG databases */ -async function dropDB(database: string) { - const config = { - host: process.env.POSTGRES_HOST, - port: process.env.POSTGRES_PORT, - user: process.env.POSTGRES_USER, - password: process.env.POSTGRES_PASSWORD, - }; - +async function dropDB(database: string, client: string) { try { - await pgtools.dropdb(config, database); + if (client === 'postgres') { + const config = { + host: process.env.POSTGRES_HOST ? process.env.POSTGRES_HOST : '', + port: process.env.POSTGRES_PORT ? process.env.POSTGRES_PORT : '', + user: process.env.POSTGRES_USER ? process.env.POSTGRES_USER : '', + password: process.env.POSTGRES_PASSWORD + ? process.env.POSTGRES_PASSWORD + : '', + }; + await pgtools.dropdb(config, database); + } else if (client === 'mysql') { + const connectionString = + process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING ?? ''; + const connection = await mysql.createConnection(connectionString); + await connection.execute('DROP DATABASE ?', [database]); + } } catch (_) { - /* do nothing*/ + /* do nothing */ } } /** Clean remnants from prior e2e runs */ -async function preCleanPostgres() { +async function preCleanDatabase() { print('Dropping old DBs'); + if (Boolean(process.env.POSTGRES_HOST)) { + await dropClientDatabases('postgres'); + } + if (Boolean(process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING)) { + await dropClientDatabases('mysql'); + } + print('Created DBs'); +} + +async function dropClientDatabases(client: string) { await Promise.all( [ 'catalog', @@ -454,9 +481,8 @@ async function preCleanPostgres() { 'proxy', 'techdocs', 'search', - ].map(name => dropDB(`backstage_plugin_${name}`)), + ].map(name => dropDB(`backstage_plugin_${name}`, client)), ); - print('Created DBs'); } /** diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js index 29e37b35f7..33fccd336d 100644 --- a/plugins/app-backend/migrations/20211229105307_init.js +++ b/plugins/app-backend/migrations/20211229105307_init.js @@ -24,7 +24,7 @@ exports.up = async function up(knex) { table.comment( 'A cache of static assets that where previously deployed and may still be lazy-loaded by clients', ); - table.text('path').primary().notNullable().comment('The path of the file'); + table.string('path').primary().notNullable().comment('The path of the file'); table .dateTime('last_modified_at') .defaultTo(knex.fn.now()) diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts index 8a0935d4e3..21e86562c6 100644 --- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts @@ -37,7 +37,7 @@ jest.setTimeout(60_000); describe('StaticAssetsStore', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); it.each(databases.eachSupportedId())( @@ -160,7 +160,11 @@ describe('StaticAssetsStore', () => { .update({ last_modified_at: knex.client.config.client.includes('sqlite3') ? knex.raw(`datetime('now', '-3600 seconds')`) - : knex.raw(`now() + interval '-3600 seconds'`), + : ( + knex.client.config.client.includes('mysql') + ? knex.raw(`date_sub(now(), interval 3600 second)`) + : knex.raw(`now() + interval '-3600 seconds'`) + ), }); expect(updated).toBe(1); diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts index 6905d63779..25606575cd 100644 --- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts @@ -144,7 +144,10 @@ export class StaticAssetsStore implements StaticAssetProvider { '<=', this.#db.client.config.client.includes('sqlite3') ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`]) - : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`), + : (this.#db.client.config.client.includes('mysql') + ? this.#db.raw(`date_sub(now(), interval ${maxAgeSeconds} second)`) + : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`) + ), ) .delete(); } diff --git a/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts b/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts index 0c78dbdd87..4e46581240 100644 --- a/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts +++ b/plugins/bazaar-backend/src/service/DatabaseHandler.test.ts @@ -36,7 +36,7 @@ jest.setTimeout(60_000); describe('DatabaseHandler', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); function createDatabaseManager( diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 0d910f664d..1f318c2590 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -24,7 +24,7 @@ import { WrapperProviders } from './WrapperProviders'; describe('WrapperProviders', () => { const applyDatabaseMigrations = jest.fn(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); const config = new ConfigReader({}); const logger = getVoidLogger(); diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index 1c8081c0ef..f600667727 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -37,7 +37,7 @@ exports.up = async function up(knex) { .notNullable() .comment('The unprocessed entity (in original form) as JSON'); table - .text('processed_entity') + .text('processed_entity', 'longtext') .nullable() .comment('The processed entity (not yet stitched) as JSON'); table @@ -83,7 +83,7 @@ exports.up = async function up(knex) { .notNullable() .comment('Random value representing a unique stitch attempt ticket'); table - .text('final_entity') + .text('final_entity', 'longtext') .nullable() .comment('The JSON encoded final entity'); table.index('entity_id', 'final_entities_entity_id_idx'); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 4959a33eef..61a062bc4a 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -138,6 +138,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { type, }), ); + await tx.batchInsert( 'relations', this.deduplicateRelations(relationRows), diff --git a/plugins/code-coverage-backend/migrations/20210302_init.js b/plugins/code-coverage-backend/migrations/20210302_init.js index a794b6a60c..35eb91541b 100644 --- a/plugins/code-coverage-backend/migrations/20210302_init.js +++ b/plugins/code-coverage-backend/migrations/20210302_init.js @@ -34,7 +34,7 @@ exports.up = async function up(knex) { .comment('An insert counter to ensure ordering'); table.uuid('id').notNullable().comment('The ID of the code coverage'); table - .text('entity') + .string('entity') .notNullable() .comment('The entity ref that this code coverage applies to'); table diff --git a/plugins/linguist-backend/migrations/20221115_init.js b/plugins/linguist-backend/migrations/20221115_init.js index 3bcf99e08b..359f7c8359 100644 --- a/plugins/linguist-backend/migrations/20221115_init.js +++ b/plugins/linguist-backend/migrations/20221115_init.js @@ -32,7 +32,7 @@ exports.up = async function up(knex) { .comment('An insert counter to ensure ordering'); table.uuid('id').notNullable().comment('The ID of the Linguist result'); table - .text('entity_ref') + .string('entity_ref') .unique() .notNullable() .comment('The entity ref that this Linguist result applies to'); diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js index 6094daa4b7..2521f6d8bb 100644 --- a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js +++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js @@ -25,7 +25,7 @@ exports.up = async function up(knex) { 'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.', ); table - .text('id') + .string('id') .notNullable() .comment('Identifier of the fact retriever plugin/package'); table diff --git a/plugins/tech-insights-backend/migrations/202109061212_facts.js b/plugins/tech-insights-backend/migrations/202109061212_facts.js index 0bdcdc0da1..237d826869 100644 --- a/plugins/tech-insights-backend/migrations/202109061212_facts.js +++ b/plugins/tech-insights-backend/migrations/202109061212_facts.js @@ -25,7 +25,7 @@ exports.up = async function up(knex) { 'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.', ); table - .text('id') + .string('id') .notNullable() .comment('Unique identifier of the fact retriever plugin/package'); table @@ -40,7 +40,7 @@ exports.up = async function up(knex) { .notNullable() .comment('The timestamp when this entry was created'); table - .text('entity') + .string('entity') .notNullable() .comment('Identifier of the entity these facts relate to'); table diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index 5bf05cda98..00b6672d7b 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -118,7 +118,7 @@ describe('FactRetrieverEngine', () => { } const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'], }); async function createEngine( diff --git a/yarn.lock b/yarn.lock index ec0379375a..7dd2088620 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23672,6 +23672,7 @@ __metadata: handlebars: ^4.7.3 nodemon: ^3.0.1 pgtools: ^1.0.0 + mysql2: ^2.2.5 puppeteer: ^17.0.0 tree-kill: ^1.2.2 ts-node: ^10.0.0 @@ -25272,6 +25273,7 @@ __metadata: express-prom-bundle: ^6.3.6 express-promise-router: ^4.1.0 luxon: ^3.0.0 + mysql2: ^2.2.5 pg: ^8.3.0 pg-connection-string: ^2.3.0 prom-client: ^14.0.1 @@ -34812,6 +34814,13 @@ __metadata: languageName: node linkType: hard +"pg-connection-string@npm:^2.6.2": + version: 2.6.2 + resolution: "pg-connection-string@npm:2.6.2" + checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5 + languageName: node + linkType: hard + "pg-int8@npm:1.0.1": version: 1.0.1 resolution: "pg-int8@npm:1.0.1" @@ -34894,6 +34903,30 @@ __metadata: languageName: node linkType: hard +"pg@npm:^8.4.0": + version: 8.11.2 + resolution: "pg@npm:8.11.2" + dependencies: + buffer-writer: 2.0.0 + packet-reader: 1.0.0 + pg-cloudflare: ^1.1.1 + pg-connection-string: ^2.6.2 + pg-pool: ^3.6.1 + pg-protocol: ^1.6.0 + pg-types: ^2.1.0 + pgpass: 1.x + peerDependencies: + pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true + peerDependenciesMeta: + pg-native: + optional: true + checksum: dfea8a2269d500dee8c17291207e5a25897163480037beb7a59be35f51e33b519c297c943ea6898b285d6a74a0d661901dc9cff2e587cc4be0bbf09b833a71a5 + languageName: node + linkType: hard + "pgpass@npm:1.x": version: 1.0.2 resolution: "pgpass@npm:1.0.2"