paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

Merge pull request #18921 from PeteLevineA/continued-mysql-support

Browse Source 
patch: Add Continued MySQL Support
This commit is contained in:
Fredrik Adelöw
2023-08-22 13:23:03 +02:00
committed by GitHub
parent 658b24e94c e57d7188cd
commit 20def974d5
28 changed files with 172 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/package.json
+1
View File
@@ -92,6 +92,7 @@
"minimatch": "^5.0.0",
"minimist": "^1.2.5",
"morgan": "^1.10.0",
"mysql2": "^2.2.5",
"node-fetch": "^2.6.7",
"node-forge": "^1.3.1",
"pg": "^8.3.0",
packages/backend-common/src/database/DatabaseManager.test.ts
+27
View File
@@ -425,6 +425,33 @@ describe('DatabaseManager', () => {
);
});
it('generates a database name override when prefix is not explicitly set for mysql', async () => {
const testManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'mysql',
connection: {
host: 'localhost',
user: 'foo',
password: 'bar',
database: 'foodb',
},
},
},
}),
);
await testManager.forPlugin('testplugin').getClient();
const mockCalls = mocked(createDatabaseClient).mock.calls.splice(-1);
const [_baseConfig, overrides] = mockCalls[0];
expect(overrides).toHaveProperty(
'connection.database',
expect.stringContaining('backstage_plugin_'),
);
});
it('uses values from plugin connection string if top level client should be used', async () => {
const pluginId = 'stringoverride';
await manager.forPlugin(pluginId).getClient();
packages/backend-tasks/migrations/20210928160613_init.js
+1 -1
View File
@@ -26,7 +26,7 @@ exports.up = async function up(knex) {
await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
table.comment('Tasks used for scheduling work on multiple workers');
table
.text('id')
.string('id')
.primary()
.notNullable()
.comment('The unique ID of this particular task');
packages/backend-tasks/src/migrations.test.ts
+1 -1
View File
@@ -43,7 +43,7 @@ jest.setTimeout(60_000);
describe('migrations', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
ids: ['POSTGRES_13', 'POSTGRES_9', 'MYSQL_8', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
packages/backend-tasks/src/tasks/PluginTaskSchedulerJanitor.test.ts
+1
View File
@@ -42,6 +42,7 @@ describe('PluginTaskSchedulerJanitor', () => {
'POSTGRES_13',
'POSTGRES_9',
'SQLITE_3',
'MYSQL_8',
],
});
packages/backend-tasks/src/tasks/TaskScheduler.test.ts
+1 -1
View File
@@ -25,7 +25,7 @@ jest.setTimeout(60_000);
describe('TaskScheduler', () => {
const logger = getVoidLogger();
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
});
async function createDatabase(
packages/backend-tasks/src/tasks/TaskWorker.test.ts
+1 -1
View File
@@ -28,7 +28,7 @@ jest.setTimeout(60_000);
describe('TaskWorker', () => {
const logger = getVoidLogger();
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3', 'MYSQL_8'],
});
beforeEach(() => {
packages/backend-tasks/src/tasks/TaskWorker.ts
+18 -9
View File
@@ -175,11 +175,15 @@ export class TaskWorker {
const time = new CronTime(settings.cadence)
.sendAt()
.minus({ seconds: 1 }) // immediately, if "* * * * * *"
.toUTC()
.toISO();
startAt = this.knex.client.config.client.includes('sqlite3')
? this.knex.raw('datetime(?)', [time])
: this.knex.raw(`?`, [time]);
.toUTC();
if (this.knex.client.config.client.includes('sqlite3')) {
startAt = this.knex.raw('datetime(?)', [time.toISO()]);
} else if (this.knex.client.config.client.includes('mysql')) {
startAt = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
} else {
startAt = this.knex.raw(`?`, [time.toISO()]);
}
} else {
startAt = this.knex.fn.now();
}
@@ -279,11 +283,16 @@ export class TaskWorker {
let nextRun: Knex.Raw;
if (isCron) {
const time = new CronTime(settings.cadence).sendAt().toUTC().toISO();
const time = new CronTime(settings.cadence).sendAt().toUTC();
this.logger.debug(`task: ${this.taskId} will next occur around ${time}`);
nextRun = this.knex.client.config.client.includes('sqlite3')
? this.knex.raw('datetime(?)', [time])
: this.knex.raw(`?`, [time]);
if (this.knex.client.config.client.includes('sqlite3')) {
nextRun = this.knex.raw('datetime(?)', [time.toISO()]);
} else if (this.knex.client.config.client.includes('mysql')) {
nextRun = this.knex.raw(`?`, [time.toSQL({ includeOffset: false })]);
} else {
nextRun = this.knex.raw(`?`, [time.toISO()]);
}
} else {
const dt = Duration.fromISO(settings.cadence).as('seconds');
this.logger.debug(
packages/backend/package.json
+1
View File
@@ -87,6 +87,7 @@
"express-prom-bundle": "^6.3.6",
"express-promise-router": "^4.1.0",
"luxon": "^3.0.0",
"mysql2": "^2.2.5",
"pg": "^8.3.0",
"pg-connection-string": "^2.3.0",
"prom-client": "^14.0.1",
packages/e2e-test/package.json
+1
View File
@@ -35,6 +35,7 @@
"cross-fetch": "^3.1.5",
"fs-extra": "10.1.0",
"handlebars": "^4.7.3",
"mysql2": "^2.2.5",
"pgtools": "^1.0.0",
"puppeteer": "^17.0.0",
"tree-kill": "^1.2.2"
packages/e2e-test/src/commands/run.ts
+37 -17
View File
@@ -31,7 +31,10 @@ import {
waitForExit,
print,
} from '../lib/helpers';
import mysql from 'mysql2/promise';
import pgtools from 'pgtools';
import { findPaths } from '@backstage/cli-common';
// eslint-disable-next-line no-restricted-syntax
@@ -66,9 +69,12 @@ export async function run() {
print('Starting the app');
await testAppServe(pluginId, appDir);
if (Boolean(process.env.POSTGRES_USER)) {
print('Testing the PostgreSQL backend startup');
await preCleanPostgres();
if (
Boolean(process.env.POSTGRES_USER) ||
Boolean(process.env.MYSQL_CONNECTION)
) {
print('Testing the database backend startup');
await preCleanDatabase();
const appConfig = path.resolve(appDir, 'app-config.yaml');
const productionConfig = path.resolve(appDir, 'app-config.production.yaml');
await testBackendStart(
@@ -79,7 +85,7 @@ export async function run() {
productionConfig,
);
}
print('Testing the SQLite backend startup');
print('Testing the Database backend startup');
await testBackendStart(appDir);
if (process.env.CI) {
@@ -427,24 +433,39 @@ async function testAppServe(pluginId: string, appDir: string) {
}
/** Drops PG databases */
async function dropDB(database: string) {
const config = {
host: process.env.POSTGRES_HOST,
port: process.env.POSTGRES_PORT,
user: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD,
};
async function dropDB(database: string, client: string) {
try {
await pgtools.dropdb(config, database);
if (client === 'postgres') {
const config = {
host: process.env.POSTGRES_HOST,
port: process.env.POSTGRES_PORT,
user: process.env.POSTGRES_USER,
password: process.env.POSTGRES_PASSWORD,
};
await pgtools.dropdb(config, database);
} else if (client === 'mysql') {
const connectionString = process.env.MYSQL_CONNECTION ?? '';
const connection = await mysql.createConnection(connectionString);
await connection.execute('DROP DATABASE ?', [database]);
}
} catch (_) {
/* do nothing*/
/* do nothing */
}
}
/** Clean remnants from prior e2e runs */
async function preCleanPostgres() {
async function preCleanDatabase() {
print('Dropping old DBs');
if (Boolean(process.env.POSTGRES_HOST)) {
await dropClientDatabases('postgres');
}
if (Boolean(process.env.MYSQL_CONNECTION)) {
await dropClientDatabases('mysql');
}
print('Dropped DBs');
}
async function dropClientDatabases(client: string) {
await Promise.all(
[
'catalog',
@@ -454,9 +475,8 @@ async function preCleanPostgres() {
'proxy',
'techdocs',
'search',
].map(name => dropDB(`backstage_plugin_${name}`)),
].map(name => dropDB(`backstage_plugin_${name}`, client)),
);
print('Created DBs');
}
/**