paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored TestDatabases to no longer depend on backend-common

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2024-05-24 16:27:54 +02:00
parent e6fb2dc950
commit 0634fdcfea
20 changed files with 836 additions and 684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/nervous-queens-fly.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-test-utils': patch
---
Refactored `TestDatabases` to no longer depend on `backend-common`
.changeset/pretty-berries-live.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/backend-defaults': patch
'@backstage/backend-common': patch
---
Deprecated `dropDatabase`
packages/backend-common/api-report.md
+1 -1
View File
@@ -312,7 +312,7 @@ export class DockerContainerRunner implements ContainerRunner {
runContainer(options: RunContainerOptions): Promise<void>;
}
// @public
// @public @deprecated
export function dropDatabase(
dbConfig: Config,
...databaseNames: string[]
packages/backend-defaults/api-report-database.md
+1 -1
View File
@@ -38,7 +38,7 @@ export const databaseServiceFactory: () => ServiceFactory<
'plugin'
>;
// @public
// @public @deprecated
export function dropDatabase(
dbConfig: Config,
...databaseNames: string[]
packages/backend-defaults/src/entrypoints/database/DatabaseManager.ts
+2 -1
View File
@@ -234,9 +234,10 @@ export class DatabaseManager implements LegacyRootDatabaseService {
}
/**
* Helper for deleting databases, only exists for backend-test-utils for now.
* Helper for deleting databases.
*
* @public
* @deprecated Will be removed in a future release.
*/
export async function dropDatabase(
dbConfig: Config,
packages/backend-test-utils/package.json
+3 -2
View File
@@ -46,7 +46,6 @@
},
"dependencies": {
"@backstage/backend-app-api": "workspace:^",
"@backstage/backend-common": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
@@ -65,9 +64,11 @@
"msw": "^1.0.0",
"mysql2": "^3.0.0",
"pg": "^8.11.3",
"pg-connection-string": "^2.3.0",
"testcontainers": "^10.0.0",
"textextensions": "^5.16.0",
"uuid": "^9.0.0"
"uuid": "^9.0.0",
"yn": "^4.0.0"
},
"devDependencies": {
"@backstage/cli": "workspace:^",
packages/backend-test-utils/src/database/TestDatabases.test.ts
-267
View File
@@ -14,28 +14,11 @@
* limitations under the License.
*/
import knexFactory from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
import { startPostgresContainer } from './startPostgresContainer';
import { TestDatabases } from './TestDatabases';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
jest.setTimeout(60_000);
describe('TestDatabases', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterAll(() => {
process.env = OLD_ENV;
});
describe('each create', () => {
const dbs = TestDatabases.create();
@@ -55,254 +38,4 @@ describe('TestDatabases', () => {
},
);
});
describe('each connect', () => {
const dbs = TestDatabases.create();
itIfDocker(
'obeys a provided connection string for postgres 16',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:16');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES16_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_16');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 15',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:15');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES15_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_15');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 14',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:14');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES14_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_14');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 13',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:13');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_13');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 12',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:12');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES12_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_12');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 11',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:11');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES11_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_11');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker(
'obeys a provided connection string for postgres 9',
async () => {
const { host, port, user, password, stop } =
await startPostgresContainer('postgres:9');
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_9');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
);
itIfDocker('obeys a provided connection string for mysql 8', async () => {
const { host, port, user, password, stop } = await startMysqlContainer(
'mysql:8',
);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING = `mysql://${user}:${password}@${host}:${port}/ignored`;
const input = await dbs.init('MYSQL_8');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = input.client.config.connection.database;
const output = knexFactory({
client: 'mysql2',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
});
});
});
packages/backend-test-utils/src/database/TestDatabases.ts
+31 -186
View File
@@ -14,27 +14,18 @@
* limitations under the License.
*/
import { DatabaseManager, dropDatabase } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { randomBytes } from 'crypto';
import { Knex } from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
import { startPostgresContainer } from './startPostgresContainer';
import { MysqlEngine } from './mysql';
import { PostgresEngine } from './postgres';
import { SqliteEngine } from './sqlite';
import {
allDatabases,
Instance,
Engine,
TestDatabaseId,
TestDatabaseProperties,
allDatabases,
} from './types';
const LARGER_POOL_CONFIG = {
pool: {
min: 0,
max: 50,
},
};
/**
* Encapsulates the creation of ephemeral test database instances for use
* inside unit or integration tests.
@@ -42,7 +33,17 @@ const LARGER_POOL_CONFIG = {
* @public
*/
export class TestDatabases {
private readonly instanceById: Map<string, Instance>;
private readonly engineFactoryByDriver: Record<
string,
(properties: TestDatabaseProperties) => Promise<Engine>
> = {
pg: PostgresEngine.create,
mysql: MysqlEngine.create,
mysql2: MysqlEngine.create,
'better-sqlite3': SqliteEngine.create,
sqlite3: SqliteEngine.create,
};
private readonly engineByTestDatabaseId: Map<string, Engine>;
private readonly supportedIds: TestDatabaseId[];
private static defaultIds?: TestDatabaseId[];
@@ -114,7 +115,7 @@ export class TestDatabases {
}
private constructor(supportedIds: TestDatabaseId[]) {
this.instanceById = new Map();
this.engineByTestDatabaseId = new Map();
this.supportedIds = supportedIds;
}
@@ -148,185 +149,29 @@ export class TestDatabases {
);
}
let instance: Instance | undefined = this.instanceById.get(id);
// Ensure that a testcontainers instance is up for this ID
if (!instance) {
instance = await this.initAny(properties);
this.instanceById.set(id, instance);
}
// Ensure that a unique logical database is created in the instance
const databaseName = `db${randomBytes(16).toString('hex')}`;
const connection = await instance.databaseManager
.forPlugin(databaseName)
.getClient();
instance.connections.push(connection);
instance.databaseNames.push(databaseName);
return connection;
}
private async initAny(properties: TestDatabaseProperties): Promise<Instance> {
// Use the connection string if provided
if (properties.driver === 'pg' || properties.driver === 'mysql2') {
const envVarName = properties.connectionStringEnvironmentVariableName;
if (envVarName) {
const connectionString = process.env[envVarName];
if (connectionString) {
const config = new ConfigReader({
backend: {
database: {
knexConfig: properties.driver.includes('sqlite')
? {}
: LARGER_POOL_CONFIG,
client: properties.driver,
connection: connectionString,
},
},
});
const databaseManager = DatabaseManager.fromConfig(config);
const databaseNames: Array<string> = [];
return {
dropDatabases: async () => {
await dropDatabase(
config.getConfig('backend.database'),
...databaseNames.map(
databaseName => `backstage_plugin_${databaseName}`,
),
);
},
databaseManager,
databaseNames,
connections: [],
};
}
}
}
// Otherwise start a container for the purpose
switch (properties.driver) {
case 'pg':
return this.initPostgres(properties);
case 'mysql2':
return this.initMysql(properties);
case 'better-sqlite3':
case 'sqlite3':
return this.initSqlite(properties);
default:
let engine = this.engineByTestDatabaseId.get(id);
if (!engine) {
const factory = this.engineFactoryByDriver[properties.driver];
if (!factory) {
throw new Error(`Unknown database driver ${properties.driver}`);
}
engine = await factory(properties);
this.engineByTestDatabaseId.set(id, engine);
}
}
private async initPostgres(
properties: TestDatabaseProperties,
): Promise<Instance> {
const { host, port, user, password, stop } = await startPostgresContainer(
properties.dockerImageName!,
);
const databaseManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
knexConfig: LARGER_POOL_CONFIG,
client: 'pg',
connection: { host, port, user, password },
},
},
}),
);
return {
stopContainer: stop,
databaseManager,
databaseNames: [],
connections: [],
};
}
private async initMysql(
properties: TestDatabaseProperties,
): Promise<Instance> {
const { host, port, user, password, stop } = await startMysqlContainer(
properties.dockerImageName!,
);
const databaseManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
knexConfig: LARGER_POOL_CONFIG,
client: 'mysql2',
connection: { host, port, user, password },
},
},
}),
);
return {
stopContainer: stop,
databaseManager,
databaseNames: [],
connections: [],
};
}
private async initSqlite(
properties: TestDatabaseProperties,
): Promise<Instance> {
const databaseManager = DatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: properties.driver,
connection: ':memory:',
},
},
}),
);
return {
databaseManager,
databaseNames: [],
connections: [],
};
return await engine.createDatabaseInstance();
}
private async shutdown() {
const instances = [...this.instanceById.values()];
this.instanceById.clear();
const engines = [...this.engineByTestDatabaseId.values()];
this.engineByTestDatabaseId.clear();
for (const {
stopContainer,
dropDatabases,
connections,
databaseManager,
} of instances) {
for (const connection of connections) {
try {
await connection.destroy();
} catch (error) {
console.warn(`TestDatabases: Failed to destroy connection`, {
connection,
error,
});
}
}
// If the database is not running in docker then drop the databases
for (const engine of engines) {
try {
await dropDatabases?.();
await engine.shutdown();
} catch (error) {
console.warn(`TestDatabases: Failed to drop databases`, {
error,
});
}
try {
await stopContainer?.();
} catch (error) {
console.warn(`TestDatabases: Failed to stop container`, {
databaseManager,
console.warn(`TestDatabases: Failed to shutdown engine`, {
engine,
error,
});
}
packages/backend-test-utils/src/database/mysql.test.ts
+113
View File
@@ -0,0 +1,113 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knexFactory, { Knex } from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { MysqlEngine, startMysqlContainer } from './mysql';
import { Engine, TestDatabaseId, allDatabases } from './types';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
const ourDatabaseIds = Object.entries(allDatabases)
.filter(([, properties]) => properties.driver.includes('mysql'))
.map(([id]) => id as TestDatabaseId);
jest.setTimeout(60_000);
describe('startMysqlContainer', () => {
itIfDocker(
'successfully launches the container and can stop it without problems',
async () => {
const { connection, stopContainer } = await startMysqlContainer(
'mysql:8',
);
const db = knexFactory({ client: 'mysql2', connection });
try {
const result = await db.select(db.raw('version() AS version'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('8.');
} finally {
await db.destroy();
await stopContainer();
}
},
);
});
describe('MysqlEngine', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterAll(() => {
process.env = OLD_ENV;
});
itIfDocker.each(ourDatabaseIds)(
'uses given connection string, %p',
async testDatabaseId => {
const properties = allDatabases[testDatabaseId];
const { connection } = await startMysqlContainer(
properties.dockerImageName!,
);
const outerKnex = knexFactory({ client: properties.driver, connection });
const databases = await outerKnex
.raw('SHOW DATABASES')
.then(rows => rows[0].length); // account for meta databases, if any
let knex: Knex | undefined;
let engine: Engine | undefined;
try {
process.env[
properties.connectionStringEnvironmentVariableName!
] = `mysql://${connection.user}:${connection.password}@${connection.host}:${connection.port}/ignored`;
engine = await MysqlEngine.create(properties);
knex = await engine.createDatabaseInstance();
// eslint-disable-next-line jest/no-standalone-expect
await expect(
outerKnex.raw('SHOW DATABASES').then(rows => rows[0].length),
).resolves.toBe(databases + 1);
} finally {
await outerKnex.destroy();
await knex?.destroy();
await engine?.shutdown();
}
},
);
itIfDocker.each(ourDatabaseIds)(
'creates docker containers, %p',
async testDatabaseId => {
const properties = allDatabases[testDatabaseId];
delete process.env[properties.connectionStringEnvironmentVariableName!];
const engine = await MysqlEngine.create(properties);
try {
const knex = await engine.createDatabaseInstance();
// eslint-disable-next-line jest/no-standalone-expect
await expect(
knex.select(knex.raw('version() as version')),
).resolves.toEqual([{ version: expect.any(String) }]);
} finally {
await engine.shutdown();
}
},
);
});
packages/backend-test-utils/src/database/mysql.ts
+242
View File
@@ -0,0 +1,242 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { stringifyError } from '@backstage/errors';
import { randomBytes } from 'crypto';
import knexFactory, { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
import yn from 'yn';
import { Engine, LARGER_POOL_CONFIG, TestDatabaseProperties } from './types';
async function waitForMysqlReady(
connection: Knex.MySqlConnectionConfig,
): Promise<void> {
const startTime = Date.now();
let lastError: Error | undefined;
let attempts = 0;
for (;;) {
attempts += 1;
let knex: Knex | undefined;
try {
knex = knexFactory({
client: 'mysql2',
connection: {
// make a copy because the driver mutates this
...connection,
},
});
const result = await knex.select(knex.raw('version() AS version'));
if (Array.isArray(result) && result[0]?.version) {
return;
}
} catch (e) {
lastError = e;
} finally {
await knex?.destroy();
}
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${attempts} attempts, ${
lastError
? `last error was ${stringifyError(lastError)}`
: '(no errors thrown)'
}`,
);
}
await new Promise(resolve => setTimeout(resolve, 100));
}
}
export async function startMysqlContainer(image: string): Promise<{
connection: Knex.MySqlConnectionConfig;
stopContainer: () => Promise<void>;
}> {
const user = 'root';
const password = uuid();
// Lazy-load to avoid side-effect of importing testcontainers
const { GenericContainer } = await import('testcontainers');
const container = await new GenericContainer(image)
.withExposedPorts(3306)
.withEnvironment({ MYSQL_ROOT_PASSWORD: password })
.withTmpFs({ '/var/lib/mysql': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(3306);
const connection = { host, port, user, password };
const stopContainer = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForMysqlReady(connection);
return { connection, stopContainer };
}
export function parseMysqlConnectionString(
connectionString: string,
): Knex.MySqlConnectionConfig {
try {
const {
protocol,
username,
password,
port,
hostname,
pathname,
searchParams,
} = new URL(connectionString);
if (protocol !== 'mysql:') {
throw new Error(`Unknown protocol ${protocol}`);
} else if (!username || !password) {
throw new Error(`Missing username/password`);
} else if (!pathname.match(/^\/[^/]+$/)) {
throw new Error(`Expected single path segment`);
}
const result: Knex.MySqlConnectionConfig = {
user: username,
password,
host: hostname,
port: Number(port || 3306),
database: decodeURIComponent(pathname.substring(1)),
};
const ssl = searchParams.get('ssl');
if (ssl) {
result.ssl = ssl;
}
const debug = searchParams.get('debug');
if (debug) {
result.debug = yn(debug);
}
return result;
} catch (e) {
throw new Error(`Error while parsing MySQL connection string, ${e}`, e);
}
}
export class MysqlEngine implements Engine {
static async create(
properties: TestDatabaseProperties,
): Promise<MysqlEngine> {
const { connectionStringEnvironmentVariableName, dockerImageName } =
properties;
if (connectionStringEnvironmentVariableName) {
const connectionString =
process.env[connectionStringEnvironmentVariableName];
if (connectionString) {
const connection = parseMysqlConnectionString(connectionString);
return new MysqlEngine(
properties,
connection as Knex.MySqlConnectionConfig,
);
}
}
if (dockerImageName) {
const { connection, stopContainer } = await startMysqlContainer(
dockerImageName,
);
return new MysqlEngine(properties, connection, stopContainer);
}
throw new Error(`Test databasee for ${properties.name} not configured`);
}
readonly #properties: TestDatabaseProperties;
readonly #connection: Knex.MySqlConnectionConfig;
readonly #knexInstances: Knex[];
readonly #databaseNames: string[];
readonly #stopContainer?: () => Promise<void>;
constructor(
properties: TestDatabaseProperties,
connection: Knex.MySqlConnectionConfig,
stopContainer?: () => Promise<void>,
) {
this.#properties = properties;
this.#connection = connection;
this.#knexInstances = [];
this.#databaseNames = [];
this.#stopContainer = stopContainer;
}
async createDatabaseInstance(): Promise<Knex> {
const adminConnection = this.#connectAdmin();
try {
const databaseName = `db${randomBytes(16).toString('hex')}`;
await adminConnection.raw('CREATE DATABASE ??', [databaseName]);
this.#databaseNames.push(databaseName);
const knexInstance = knexFactory({
client: this.#properties.driver,
connection: {
...this.#connection,
database: databaseName,
},
...LARGER_POOL_CONFIG,
});
this.#knexInstances.push(knexInstance);
return knexInstance;
} finally {
await adminConnection.destroy();
}
}
async shutdown(): Promise<void> {
for (const instance of this.#knexInstances) {
await instance.destroy();
}
const adminConnection = this.#connectAdmin();
try {
for (const databaseName of this.#databaseNames) {
await adminConnection.raw('DROP DATABASE ??', [databaseName]);
}
} finally {
await adminConnection.destroy();
}
await this.#stopContainer?.();
}
#connectAdmin(): Knex {
const connection = {
...this.#connection,
database: null as unknown as string,
};
return knexFactory({
client: this.#properties.driver,
connection,
pool: {
acquireTimeoutMillis: 10000,
},
});
}
}
packages/backend-test-utils/src/database/postgres.test.ts
+113
View File
@@ -0,0 +1,113 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knexFactory, { Knex } from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startPostgresContainer, PostgresEngine } from './postgres';
import { Engine, TestDatabaseId, allDatabases } from './types';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
const ourDatabaseIds = Object.entries(allDatabases)
.filter(([, properties]) => properties.driver === 'pg')
.map(([id]) => id as TestDatabaseId);
jest.setTimeout(60_000);
describe('startPostgresContainer', () => {
itIfDocker(
'successfully launches the container and can stop it without problems',
async () => {
const { connection, stopContainer } = await startPostgresContainer(
'postgres:13',
);
const db = knexFactory({ client: 'pg', connection });
try {
const result = await db.select(db.raw('version()'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('PostgreSQL');
} finally {
await db.destroy();
await stopContainer();
}
},
);
});
describe('PostgresEngine', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterAll(() => {
process.env = OLD_ENV;
});
itIfDocker.each(ourDatabaseIds)(
'uses given connection string, %p',
async testDatabaseId => {
const properties = allDatabases[testDatabaseId];
const { connection } = await startPostgresContainer(
properties.dockerImageName!,
);
const outerKnex = knexFactory({ client: properties.driver, connection });
const databases = await outerKnex
.from('pg_database')
.then(rows => rows.length); // account for postgres, template0 etc
let knex: Knex | undefined;
let engine: Engine | undefined;
try {
process.env[
properties.connectionStringEnvironmentVariableName!
] = `postgres://${connection.user}:${connection.password}@${connection.host}:${connection.port}`;
engine = await PostgresEngine.create(properties);
knex = await engine.createDatabaseInstance();
// eslint-disable-next-line jest/no-standalone-expect
await expect(outerKnex.from('pg_database')).resolves.toHaveLength(
databases + 1,
);
} finally {
await outerKnex.destroy();
await knex?.destroy();
await engine?.shutdown();
}
},
);
itIfDocker.each(ourDatabaseIds)(
'creates docker containers, %p',
async testDatabaseId => {
const properties = allDatabases[testDatabaseId];
delete process.env[properties.connectionStringEnvironmentVariableName!];
const engine = await PostgresEngine.create(properties);
try {
const knex = await engine.createDatabaseInstance();
// eslint-disable-next-line jest/no-standalone-expect
await expect(knex.select(knex.raw('version()'))).resolves.toEqual([
{ version: expect.stringContaining('PostgreSQL') },
]);
} finally {
await engine.shutdown();
}
},
);
});
packages/backend-test-utils/src/database/postgres.ts
+195
View File
@@ -0,0 +1,195 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { stringifyError } from '@backstage/errors';
import { randomBytes } from 'crypto';
import knexFactory, { Knex } from 'knex';
import { parse as parsePgConnectionString } from 'pg-connection-string';
import { v4 as uuid } from 'uuid';
import { Engine, LARGER_POOL_CONFIG, TestDatabaseProperties } from './types';
async function waitForPostgresReady(
connection: Knex.PgConnectionConfig,
): Promise<void> {
const startTime = Date.now();
let lastError: Error | undefined;
let attempts = 0;
for (;;) {
attempts += 1;
let knex: Knex | undefined;
try {
knex = knexFactory({
client: 'pg',
connection: {
// make a copy because the driver mutates this
...connection,
},
});
const result = await knex.select(knex.raw('version()'));
if (Array.isArray(result) && result[0]?.version) {
return;
}
} catch (e) {
lastError = e;
} finally {
await knex?.destroy();
}
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${attempts} attempts, ${
lastError
? `last error was ${stringifyError(lastError)}`
: '(no errors thrown)'
}`,
);
}
await new Promise(resolve => setTimeout(resolve, 100));
}
}
export async function startPostgresContainer(image: string): Promise<{
connection: Knex.PgConnectionConfig;
stopContainer: () => Promise<void>;
}> {
const user = 'postgres';
const password = uuid();
// Lazy-load to avoid side-effect of importing testcontainers
const { GenericContainer } = await import('testcontainers');
const container = await new GenericContainer(image)
.withExposedPorts(5432)
.withEnvironment({ POSTGRES_PASSWORD: password })
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(5432);
const connection = { host, port, user, password };
const stopContainer = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForPostgresReady(connection);
return { connection, stopContainer };
}
export class PostgresEngine implements Engine {
static async create(
properties: TestDatabaseProperties,
): Promise<PostgresEngine> {
const { connectionStringEnvironmentVariableName, dockerImageName } =
properties;
if (connectionStringEnvironmentVariableName) {
const connectionString =
process.env[connectionStringEnvironmentVariableName];
if (connectionString) {
const connection = parsePgConnectionString(connectionString);
return new PostgresEngine(
properties,
connection as Knex.PgConnectionConfig,
);
}
}
if (dockerImageName) {
const { connection, stopContainer } = await startPostgresContainer(
dockerImageName,
);
return new PostgresEngine(properties, connection, stopContainer);
}
throw new Error(`Test databasee for ${properties.name} not configured`);
}
readonly #properties: TestDatabaseProperties;
readonly #connection: Knex.PgConnectionConfig;
readonly #knexInstances: Knex[];
readonly #databaseNames: string[];
readonly #stopContainer?: () => Promise<void>;
constructor(
properties: TestDatabaseProperties,
connection: Knex.PgConnectionConfig,
stopContainer?: () => Promise<void>,
) {
this.#properties = properties;
this.#connection = connection;
this.#knexInstances = [];
this.#databaseNames = [];
this.#stopContainer = stopContainer;
}
async createDatabaseInstance(): Promise<Knex> {
const adminConnection = this.#connectAdmin();
try {
const databaseName = `db${randomBytes(16).toString('hex')}`;
await adminConnection.raw('CREATE DATABASE ??', [databaseName]);
this.#databaseNames.push(databaseName);
const knexInstance = knexFactory({
client: this.#properties.driver,
connection: {
...this.#connection,
database: databaseName,
},
...LARGER_POOL_CONFIG,
});
this.#knexInstances.push(knexInstance);
return knexInstance;
} finally {
await adminConnection.destroy();
}
}
async shutdown(): Promise<void> {
for (const instance of this.#knexInstances) {
await instance.destroy();
}
const adminConnection = this.#connectAdmin();
try {
for (const databaseName of this.#databaseNames) {
await adminConnection.raw('DROP DATABASE ??', [databaseName]);
}
} finally {
await adminConnection.destroy();
}
await this.#stopContainer?.();
}
#connectAdmin(): Knex {
return knexFactory({
client: this.#properties.driver,
connection: {
...this.#connection,
database: 'postgres',
},
pool: {
acquireTimeoutMillis: 10000,
},
});
}
}
packages/backend-test-utils/src/database/sqlite.test.ts
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { SqliteEngine } from './sqlite';
import { TestDatabaseId, allDatabases } from './types';
const ourDatabaseIds = Object.entries(allDatabases)
.filter(([, properties]) => properties.driver.includes('sqlite'))
.map(([id]) => id as TestDatabaseId);
describe('SqliteEngine', () => {
it.each(ourDatabaseIds)(
'should create a database instance, %p',
async testDatabaseId => {
for (let i = 0; i < 100; ++i) {
const properties = allDatabases[testDatabaseId];
const engine = await SqliteEngine.create(properties);
const instance1 = await engine.createDatabaseInstance();
const instance2 = await engine.createDatabaseInstance();
expect(instance1).toBeDefined();
expect(instance2).toBeDefined();
expect(instance1).not.toEqual(instance2);
await instance1.schema.createTable('t', table => {
table.string('value');
});
await instance2.schema.createTable('t', table => {
table.string('value');
});
await instance1('t').insert({ value: 'value1' });
await instance2('t').insert({ value: 'value2' });
await expect(instance1('t')).resolves.toEqual([{ value: 'value1' }]);
await expect(instance2('t')).resolves.toEqual([{ value: 'value2' }]);
await engine.shutdown();
}
},
);
});
packages/backend-test-utils/src/database/sqlite.ts
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knexFactory, { Knex } from 'knex';
import { Engine, TestDatabaseProperties } from './types';
export class SqliteEngine implements Engine {
static async create(
properties: TestDatabaseProperties,
): Promise<SqliteEngine> {
return new SqliteEngine(properties);
}
readonly #properties: TestDatabaseProperties;
readonly #instances: Knex[];
constructor(properties: TestDatabaseProperties) {
this.#properties = properties;
this.#instances = [];
}
async createDatabaseInstance(): Promise<Knex> {
const instance = knexFactory({
client: this.#properties.driver,
connection: ':memory:',
useNullAsDefault: true,
});
instance.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
this.#instances.push(instance);
return instance;
}
async shutdown(): Promise<void> {
for (const instance of this.#instances) {
await instance.destroy();
}
}
}
packages/backend-test-utils/src/database/startMysqlContainer.test.ts
-38
View File
@@ -1,38 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
jest.setTimeout(60_000);
describe('startMysqlContainer', () => {
itIfDocker('successfully launches the container', async () => {
const { stop, ...connection } = await startMysqlContainer('mysql:8');
const db = createConnection({ client: 'mysql2', connection });
try {
const result = await db.select(db.raw('version() AS version'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('8.');
} finally {
await db.destroy();
await stop();
}
});
});
packages/backend-test-utils/src/database/startMysqlContainer.ts
-70
View File
@@ -1,70 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection, { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
async function waitForMysqlReady(
connection: Knex.MySqlConnectionConfig,
): Promise<void> {
const startTime = Date.now();
const db = createConnection({ client: 'mysql2', connection });
try {
for (;;) {
try {
const result = await db.select(db.raw('version() AS version'));
if (result[0]?.version) {
return;
}
} catch (e) {
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${e}`,
);
}
}
await new Promise(resolve => setTimeout(resolve, 100));
}
} finally {
db.destroy();
}
}
export async function startMysqlContainer(image: string) {
const user = 'root';
const password = uuid();
// Lazy-load to avoid side-effect of importing testcontainers
const { GenericContainer } = await import('testcontainers');
const container = await new GenericContainer(image)
.withExposedPorts(3306)
.withEnvironment({ MYSQL_ROOT_PASSWORD: password })
.withTmpFs({ '/var/lib/mysql': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(3306);
const stop = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForMysqlReady({ host, port, user, password });
return { host, port, user, password, stop };
}
packages/backend-test-utils/src/database/startPostgresContainer.test.ts
-38
View File
@@ -1,38 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startPostgresContainer } from './startPostgresContainer';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
jest.setTimeout(60_000);
describe('startPostgresContainer', () => {
itIfDocker('successfully launches the container', async () => {
const { stop, ...connection } = await startPostgresContainer('postgres:13');
const db = createConnection({ client: 'pg', connection });
try {
const result = await db.select(db.raw('version()'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('PostgreSQL');
} finally {
await db.destroy();
await stop();
}
});
});
packages/backend-test-utils/src/database/startPostgresContainer.ts
-70
View File
@@ -1,70 +0,0 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection, { Knex } from 'knex';
import { v4 as uuid } from 'uuid';
async function waitForPostgresReady(
connection: Knex.PgConnectionConfig,
): Promise<void> {
const startTime = Date.now();
const db = createConnection({ client: 'pg', connection });
try {
for (;;) {
try {
const result = await db.select(db.raw('version()'));
if (Array.isArray(result) && result[0]?.version) {
return;
}
} catch (e) {
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${e}`,
);
}
}
await new Promise(resolve => setTimeout(resolve, 100));
}
} finally {
db.destroy();
}
}
export async function startPostgresContainer(image: string) {
const user = 'postgres';
const password = uuid();
// Lazy-load to avoid side-effect of importing testcontainers
const { GenericContainer } = await import('testcontainers');
const container = await new GenericContainer(image)
.withExposedPorts(5432)
.withEnvironment({ POSTGRES_PASSWORD: password })
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(5432);
const stop = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForPostgresReady({ host, port, user, password });
return { host, port, user, password, stop };
}
packages/backend-test-utils/src/database/types.ts
+12 -9
View File
@@ -14,10 +14,14 @@
* limitations under the License.
*/
import { DatabaseManager } from '@backstage/backend-common';
import { Knex } from 'knex';
import { getDockerImageForName } from '../util/getDockerImageForName';
export interface Engine {
createDatabaseInstance(): Promise<Knex>;
shutdown(): Promise<void>;
}
/**
* The possible databases to test against.
*
@@ -41,14 +45,6 @@ export type TestDatabaseProperties = {
connectionStringEnvironmentVariableName?: string;
};
export type Instance = {
stopContainer?: () => Promise<void>;
dropDatabases?: () => Promise<void>;
databaseManager: DatabaseManager;
connections: Array<Knex>;
databaseNames: Array<string>;
};
export const allDatabases: Record<TestDatabaseId, TestDatabaseProperties> =
Object.freeze({
POSTGRES_16: {
@@ -112,3 +108,10 @@ export const allDatabases: Record<TestDatabaseId, TestDatabaseProperties> =
driver: 'better-sqlite3',
},
});
export const LARGER_POOL_CONFIG = {
pool: {
min: 0,
max: 50,
},
};
yarn.lock
+2 -1
View File
@@ -3574,7 +3574,6 @@ __metadata:
resolution: "@backstage/backend-test-utils@workspace:packages/backend-test-utils"
dependencies:
"@backstage/backend-app-api": "workspace:^"
"@backstage/backend-common": "workspace:^"
"@backstage/backend-plugin-api": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
@@ -3595,10 +3594,12 @@ __metadata:
msw: ^1.0.0
mysql2: ^3.0.0
pg: ^8.11.3
pg-connection-string: ^2.3.0
supertest: ^6.1.3
testcontainers: ^10.0.0
textextensions: ^5.16.0
uuid: ^9.0.0
yn: ^4.0.0
peerDependencies:
"@types/jest": "*"
languageName: unknown