diff --git a/.github/workflows/master-win.yml b/.github/workflows/master-win.yml index a9b5bfdf0e..3850146388 100644 --- a/.github/workflows/master-win.yml +++ b/.github/workflows/master-win.yml @@ -55,6 +55,8 @@ jobs: - name: test run: yarn lerna -- run test + env: + BACKSTAGE_TEST_DISABLE_DOCKER: 1 # credit: https://github.com/appleboy/discord-action/issues/3#issuecomment-731426861 - name: Discord notification diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 60db9ac16b..75e26d52d3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,6 +16,41 @@ jobs: matrix: node-version: [12.x, 14.x] + services: + postgres13: + image: postgres:13 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432/tcp + postgres9: + image: postgres:9 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432/tcp + mysql8: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: root + options: >- + --health-cmd "mysqladmin ping -h localhost" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 3306/tcp + env: CI: true NODE_OPTIONS: --max-old-space-size=4096 @@ -86,6 +121,10 @@ jobs: # Upload code coverage for some specific flags. Also see .codecov.yml bash <(curl -s https://codecov.io/bash) -f packages/core/coverage/* -F core bash <(curl -s https://codecov.io/bash) -f packages/core-api/coverage/* -F core-api + env: + BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} + BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} + BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored - name: Discord notification if: ${{ failure() }} diff --git a/packages/backend-test-utils/.eslintrc.js b/packages/backend-test-utils/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/backend-test-utils/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/backend-test-utils/README.md b/packages/backend-test-utils/README.md new file mode 100644 index 0000000000..38756fa5d7 --- /dev/null +++ b/packages/backend-test-utils/README.md @@ -0,0 +1,18 @@ +# @backstage/backend-test-utils + +Test helpers library for Backstage backends. + +## Usage + +Add the library as a `devDependency` to your backend package: + +```sh +# From the Backstage root directory, go to your backend package, or to a backend plugin +cd plugins/my-plugin-backend +yarn add --dev @backstage/backend-test-utils +``` + +## Documentation + +- [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) +- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md) diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md new file mode 100644 index 0000000000..28d4a00463 --- /dev/null +++ b/packages/backend-test-utils/api-report.md @@ -0,0 +1,31 @@ +## API Report File for "@backstage/backend-test-utils" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { Knex } from 'knex'; + +// @public (undocumented) +export function isDockerDisabledForTests(): boolean; + +// @public +export type TestDatabaseId = 'POSTGRES_13' | 'POSTGRES_9' | 'MYSQL_8' | 'SQLITE_3'; + +// @public +export class TestDatabases { + static create(options?: { + ids?: TestDatabaseId[]; + disableDocker?: boolean; + }): TestDatabases; + // (undocumented) + eachSupportedId(): [TestDatabaseId][]; + init(id: TestDatabaseId): Promise; + // (undocumented) + supports(id: TestDatabaseId): boolean; +} + + +// (No @packageDocumentation comment for this package) + +``` diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json new file mode 100644 index 0000000000..95276651ca --- /dev/null +++ b/packages/backend-test-utils/package.json @@ -0,0 +1,50 @@ +{ + "name": "@backstage/backend-test-utils", + "description": "Test helpers library for Backstage backends", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/backend-test-utils" + }, + "keywords": [ + "backstage", + "test" + ], + "license": "Apache-2.0", + "scripts": { + "build": "backstage-cli build --outputs cjs,types", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.8.0", + "@backstage/cli": "^0.6.10", + "@backstage/config": "^0.1.5", + "knex": "^0.95.1", + "mysql2": "^2.2.5", + "pg": "^8.3.0", + "sqlite3": "^5.0.0", + "testcontainers": "^7.10.0", + "uuid": "^8.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.6.10", + "jest": "^26.0.1" + }, + "files": [ + "dist" + ] +} diff --git a/packages/backend-test-utils/src/database/TestDatabases.test.ts b/packages/backend-test-utils/src/database/TestDatabases.test.ts new file mode 100644 index 0000000000..7c1e6e1bdd --- /dev/null +++ b/packages/backend-test-utils/src/database/TestDatabases.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knexFactory from 'knex'; +import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +import { startMysqlContainer } from './startMysqlContainer'; +import { startPostgresContainer } from './startPostgresContainer'; +import { TestDatabases } from './TestDatabases'; + +const itIfDocker = isDockerDisabledForTests() ? it.skip : it; + +describe('TestDatabases', () => { + const OLD_ENV = process.env; + beforeEach(() => { + jest.resetModules(); + process.env = { ...OLD_ENV }; + }); + afterAll(() => { + process.env = OLD_ENV; + }); + + describe('each', () => { + const dbs = TestDatabases.create(); + + it.each(dbs.eachSupportedId())( + 'creates distinct %p databases', + async databaseId => { + if (!dbs.supports(databaseId)) { + return; + } + const db1 = await dbs.init(databaseId); + const db2 = await dbs.init(databaseId); + await db1.schema.createTable('a', table => table.string('x').primary()); + await db2.schema.createTable('a', table => table.string('y').primary()); + await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([ + { a: 1 }, + ]); + }, + 60_000, + ); + }); + + itIfDocker( + 'obeys a provided connection string for postgres 13', + async () => { + const dbs = TestDatabases.create(); + const { host, port, user, password, stop } = await startPostgresContainer( + 'postgres:13', + ); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`; + const input = await dbs.init('POSTGRES_13'); + await input.schema.createTable('a', table => + table.string('x').primary(), + ); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const database = 'backstage_plugin_0'; + const output = knexFactory({ + client: 'pg', + connection: { host, port, user, password, database }, + }); + // eslint-disable-next-line jest/no-standalone-expect + await expect(output.select('x').from('a')).resolves.toEqual([ + { x: 'y' }, + ]); + } finally { + await stop(); + } + }, + 60_000, + ); + + itIfDocker( + 'obeys a provided connection string for postgres 9', + async () => { + const dbs = TestDatabases.create(); + const { host, port, user, password, stop } = await startPostgresContainer( + 'postgres:9', + ); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`; + const input = await dbs.init('POSTGRES_9'); + await input.schema.createTable('a', table => + table.string('x').primary(), + ); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const database = 'backstage_plugin_0'; + const output = knexFactory({ + client: 'pg', + connection: { host, port, user, password, database }, + }); + // eslint-disable-next-line jest/no-standalone-expect + await expect(output.select('x').from('a')).resolves.toEqual([ + { x: 'y' }, + ]); + } finally { + await stop(); + } + }, + 60_000, + ); + + itIfDocker( + 'obeys a provided connection string for mysql 8', + async () => { + const dbs = TestDatabases.create(); + const { host, port, user, password, stop } = await startMysqlContainer( + 'mysql:8', + ); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING = `mysql://${user}:${password}@${host}:${port}/ignored`; + const input = await dbs.init('MYSQL_8'); + await input.schema.createTable('a', table => + table.string('x').primary(), + ); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const database = 'backstage_plugin_0'; + const output = knexFactory({ + client: 'mysql2', + connection: { host, port, user, password, database }, + }); + // eslint-disable-next-line jest/no-standalone-expect + await expect(output.select('x').from('a')).resolves.toEqual([ + { x: 'y' }, + ]); + } finally { + await stop(); + } + }, + 60_000, + ); +}); diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts new file mode 100644 index 0000000000..a7b08cd593 --- /dev/null +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { Knex } from 'knex'; +import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +import { startMysqlContainer } from './startMysqlContainer'; +import { startPostgresContainer } from './startPostgresContainer'; +import { + allDatabases, + Instance, + TestDatabaseId, + TestDatabaseProperties, +} from './types'; + +/** + * Encapsulates the creation of ephemeral test database instances for use + * inside unit or integration tests. + */ +export class TestDatabases { + private readonly instanceById: Map; + private readonly supportedIds: TestDatabaseId[]; + private lastDatabaseIndex: number; + + /** + * Creates an empty `TestDatabases` instance, and sets up Jest to clean up + * all of its acquired resources after all tests finish. + * + * You typically want to create just a single instance like this at the top + * of your test file or `describe` block, and then call `init` many times on + * that instance inside the individual tests. Spinning up a "physical" + * database instance takes a considerable amount of time, slowing down tests. + * But initializing a new logical database inside that instance using `init` + * is very fast. + */ + static create(options?: { + ids?: TestDatabaseId[]; + disableDocker?: boolean; + }): TestDatabases { + const defaultOptions = { + ids: Object.keys(allDatabases) as TestDatabaseId[], + disableDocker: isDockerDisabledForTests(), + }; + + const { ids, disableDocker } = Object.assign(defaultOptions, options ?? {}); + + const supportedIds = ids.filter(id => { + const properties = allDatabases[id]; + if (!properties) { + return false; + } + // If the caller has set up the env with an explicit connection string, + // we'll assume that this database will work + if ( + properties.connectionStringEnvironmentVariableName && + process.env[properties.connectionStringEnvironmentVariableName] + ) { + return true; + } + // If the database doesn't require docker at all, there's nothing to worry + // about + if (!properties.dockerImageName) { + return true; + } + // If the database requires docker, but docker is disabled, we will fail. + if (disableDocker) { + return false; + } + return true; + }); + + const databases = new TestDatabases(supportedIds); + + afterAll(async () => { + await databases.shutdown(); + }); + + return databases; + } + + private constructor(supportedIds: TestDatabaseId[]) { + this.instanceById = new Map(); + this.supportedIds = supportedIds; + this.lastDatabaseIndex = 0; + } + + supports(id: TestDatabaseId): boolean { + return this.supportedIds.includes(id); + } + + eachSupportedId(): [TestDatabaseId][] { + return this.supportedIds.map(id => [id]); + } + + /** + * Returns a fresh, unique, empty logical database on an instance of the + * given database ID platform. + * + * @param id The ID of the database platform to use, e.g. 'POSTGRES_13' + * @returns A `Knex` connection object + */ + async init(id: TestDatabaseId): Promise { + const properties = allDatabases[id]; + if (!properties) { + const candidates = Object.keys(allDatabases).join(', '); + throw new Error( + `Unknown test database ${id}, possible values are ${candidates}`, + ); + } + if (!this.supportedIds.includes(id)) { + const candidates = this.supportedIds.join(', '); + throw new Error( + `Unsupported test database ${id} for this environment, possible values are ${candidates}`, + ); + } + + let instance: Instance | undefined = this.instanceById.get(id); + + // Ensure that a testcontainers instance is up for this ID + if (!instance) { + instance = await this.initAny(properties); + this.instanceById.set(id, instance); + } + + // Ensure that a unique logical database is created in the instance + const connection = await instance.databaseManager + .forPlugin(String(this.lastDatabaseIndex++)) + .getClient(); + + instance.connections.push(connection); + + return connection; + } + + private async initAny(properties: TestDatabaseProperties): Promise { + // Use the connection string if provided + if (properties.driver === 'pg' || properties.driver === 'mysql2') { + const envVarName = properties.connectionStringEnvironmentVariableName; + if (envVarName) { + const connectionString = process.env[envVarName]; + if (connectionString) { + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: properties.driver, + connection: connectionString, + }, + }, + }), + ); + return { + databaseManager, + connections: [], + }; + } + } + } + + // Otherwise start a container for the purpose + switch (properties.driver) { + case 'pg': + return this.initPostgres(properties); + case 'mysql2': + return this.initMysql(properties); + case 'sqlite3': + return this.initSqlite(properties); + default: + throw new Error(`Unknown database driver ${properties.driver}`); + } + } + + private async initPostgres( + properties: TestDatabaseProperties, + ): Promise { + const { host, port, user, password, stop } = await startPostgresContainer( + properties.dockerImageName!, + ); + + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: { host, port, user, password }, + }, + }, + }), + ); + + return { + stopContainer: stop, + databaseManager, + connections: [], + }; + } + + private async initMysql( + properties: TestDatabaseProperties, + ): Promise { + const { host, port, user, password, stop } = await startMysqlContainer( + properties.dockerImageName!, + ); + + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'mysql2', + connection: { host, port, user, password }, + }, + }, + }), + ); + + return { + stopContainer: stop, + databaseManager, + connections: [], + }; + } + + private async initSqlite( + _properties: TestDatabaseProperties, + ): Promise { + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ); + + return { + databaseManager, + connections: [], + }; + } + + private async shutdown() { + const instances = [...this.instanceById.values()]; + await Promise.all( + instances.map(async ({ stopContainer, connections }) => { + try { + await Promise.all(connections.map(c => c.destroy())); + } catch { + // ignore + } + try { + await stopContainer?.(); + } catch { + // ignore + } + }), + ); + } +} diff --git a/packages/backend-test-utils/src/database/index.ts b/packages/backend-test-utils/src/database/index.ts new file mode 100644 index 0000000000..6988f0b80b --- /dev/null +++ b/packages/backend-test-utils/src/database/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +export { TestDatabases } from './TestDatabases'; +export type { TestDatabaseId } from './types'; diff --git a/packages/backend-test-utils/src/database/startMysqlContainer.test.ts b/packages/backend-test-utils/src/database/startMysqlContainer.test.ts new file mode 100644 index 0000000000..e210ccb413 --- /dev/null +++ b/packages/backend-test-utils/src/database/startMysqlContainer.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import createConnection from 'knex'; +import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +import { startMysqlContainer } from './startMysqlContainer'; + +const itIfDocker = isDockerDisabledForTests() ? it.skip : it; + +describe('startMysqlContainer', () => { + itIfDocker( + 'successfully launches the container', + async () => { + const { stop, ...connection } = await startMysqlContainer('mysql:8'); + const db = createConnection({ client: 'mysql2', connection }); + try { + const result = await db.select(db.raw('version() AS version')); + // eslint-disable-next-line jest/no-standalone-expect + expect(result[0]?.version).toContain('8.'); + } finally { + await db.destroy(); + await stop(); + } + }, + 60_000, + ); +}); diff --git a/packages/backend-test-utils/src/database/startMysqlContainer.ts b/packages/backend-test-utils/src/database/startMysqlContainer.ts new file mode 100644 index 0000000000..739298dfe5 --- /dev/null +++ b/packages/backend-test-utils/src/database/startMysqlContainer.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import createConnection, { Knex } from 'knex'; +import { GenericContainer } from 'testcontainers'; +import { v4 as uuid } from 'uuid'; + +async function waitForMysqlReady( + connection: Knex.MySqlConnectionConfig, +): Promise { + const startTime = Date.now(); + const db = createConnection({ client: 'mysql2', connection }); + + try { + for (;;) { + try { + const result = await db.select(db.raw('version() AS version')); + if (result[0]?.version) { + return; + } + } catch (e) { + if (Date.now() - startTime > 30_000) { + throw new Error( + `Timed out waiting for the database to be ready for connections, ${e}`, + ); + } + } + + await new Promise(resolve => setTimeout(resolve, 100)); + } + } finally { + db.destroy(); + } +} + +export async function startMysqlContainer(image: string) { + const user = 'root'; + const password = uuid(); + + const container = await new GenericContainer(image) + .withExposedPorts(3306) + .withEnv('MYSQL_ROOT_PASSWORD', password) + .withTmpFs({ '/var/lib/mysql': 'rw' }) + .start(); + + const host = container.getHost(); + const port = container.getMappedPort(3306); + const stop = async () => { + await container.stop({ timeout: 10_000 }); + }; + + await waitForMysqlReady({ host, port, user, password }); + + return { host, port, user, password, stop }; +} diff --git a/packages/backend-test-utils/src/database/startPostgresContainer.test.ts b/packages/backend-test-utils/src/database/startPostgresContainer.test.ts new file mode 100644 index 0000000000..4e8acc8410 --- /dev/null +++ b/packages/backend-test-utils/src/database/startPostgresContainer.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import createConnection from 'knex'; +import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; +import { startPostgresContainer } from './startPostgresContainer'; + +const itIfDocker = isDockerDisabledForTests() ? it.skip : it; + +describe('startPostgresContainer', () => { + itIfDocker( + 'successfully launches the container', + async () => { + const { stop, ...connection } = await startPostgresContainer( + 'postgres:13', + ); + const db = createConnection({ client: 'pg', connection }); + try { + const result = await db.select(db.raw('version()')); + // eslint-disable-next-line jest/no-standalone-expect + expect(result[0]?.version).toContain('PostgreSQL'); + } finally { + await db.destroy(); + await stop(); + } + }, + 60_000, + ); +}); diff --git a/packages/backend-test-utils/src/database/startPostgresContainer.ts b/packages/backend-test-utils/src/database/startPostgresContainer.ts new file mode 100644 index 0000000000..1c2ecdcb60 --- /dev/null +++ b/packages/backend-test-utils/src/database/startPostgresContainer.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import createConnection, { Knex } from 'knex'; +import { GenericContainer } from 'testcontainers'; +import { v4 as uuid } from 'uuid'; + +async function waitForPostgresReady( + connection: Knex.PgConnectionConfig, +): Promise { + const startTime = Date.now(); + const db = createConnection({ client: 'pg', connection }); + + try { + for (;;) { + try { + const result = await db.select(db.raw('version()')); + if (Array.isArray(result) && result[0]?.version) { + return; + } + } catch (e) { + if (Date.now() - startTime > 30_000) { + throw new Error( + `Timed out waiting for the database to be ready for connections, ${e}`, + ); + } + } + + await new Promise(resolve => setTimeout(resolve, 100)); + } + } finally { + db.destroy(); + } +} + +export async function startPostgresContainer(image: string) { + const user = 'postgres'; + const password = uuid(); + + const container = await new GenericContainer(image) + .withExposedPorts(5432) + .withEnv('POSTGRES_PASSWORD', password) + .withTmpFs({ '/var/lib/postgresql/data': 'rw' }) + .start(); + + const host = container.getHost(); + const port = container.getMappedPort(5432); + const stop = async () => { + await container.stop({ timeout: 10_000 }); + }; + + await waitForPostgresReady({ host, port, user, password }); + + return { host, port, user, password, stop }; +} diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts new file mode 100644 index 0000000000..791cf09b7b --- /dev/null +++ b/packages/backend-test-utils/src/database/types.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { Knex } from 'knex'; + +/** + * The possible databases to test against. + */ +export type TestDatabaseId = + | 'POSTGRES_13' + | 'POSTGRES_9' + | 'MYSQL_8' + | 'SQLITE_3'; + +export type TestDatabaseProperties = { + name: string; + driver: string; + dockerImageName?: string; + connectionStringEnvironmentVariableName?: string; +}; + +export type Instance = { + stopContainer?: () => Promise; + databaseManager: SingleConnectionDatabaseManager; + connections: Array; +}; + +export const allDatabases: Record< + TestDatabaseId, + TestDatabaseProperties +> = Object.freeze({ + POSTGRES_13: { + name: 'Postgres 13.x', + driver: 'pg', + dockerImageName: 'postgres:13', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING', + }, + POSTGRES_9: { + name: 'Postgres 9.x', + driver: 'pg', + dockerImageName: 'postgres:9', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING', + }, + MYSQL_8: { + name: 'MySQL 8.x', + driver: 'mysql2', + dockerImageName: 'mysql:8', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING', + }, + SQLITE_3: { + name: 'SQLite 3.x', + driver: 'sqlite3', + }, +}); diff --git a/packages/backend-test-utils/src/index.ts b/packages/backend-test-utils/src/index.ts new file mode 100644 index 0000000000..ad3e422f44 --- /dev/null +++ b/packages/backend-test-utils/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './database'; +export * from './util'; diff --git a/packages/backend-test-utils/src/setupTests.ts b/packages/backend-test-utils/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/packages/backend-test-utils/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/packages/backend-test-utils/src/util/index.ts b/packages/backend-test-utils/src/util/index.ts new file mode 100644 index 0000000000..e4e0e97ac1 --- /dev/null +++ b/packages/backend-test-utils/src/util/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { isDockerDisabledForTests } from './isDockerDisabledForTests'; diff --git a/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts b/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts new file mode 100644 index 0000000000..60d27bc154 --- /dev/null +++ b/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isDockerDisabledForTests() { + return Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER); +} diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts index 37dc4769fd..a0d794b745 100644 --- a/scripts/api-extractor.ts +++ b/scripts/api-extractor.ts @@ -62,6 +62,7 @@ PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = function tryGetPackag const DOCUMENTED_PACKAGES = [ 'packages/backend-common', + 'packages/backend-test-utils', 'packages/catalog-client', 'packages/catalog-model', 'packages/cli-common', diff --git a/yarn.lock b/yarn.lock index f0623bee33..7a83441888 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6822,6 +6822,21 @@ resolved "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9" integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA== +"@types/ssh2-streams@*": + version "0.1.8" + resolved "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.8.tgz#142af404dae059931aea7fcd1511b5478964feb6" + integrity sha512-I7gixRPUvVIyJuCEvnmhr3KvA2dC0639kKswqD4H5b4/FOcnPtNU+qWLiXdKIqqX9twUvi5j0U1mwKE5CUsrfA== + dependencies: + "@types/node" "*" + +"@types/ssh2@^0.5.45": + version "0.5.46" + resolved "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.46.tgz#e12341a242aea0e98ac2dec89e039bf421fd3584" + integrity sha512-1pC8FHrMPYdkLoUOwTYYifnSEPzAFZRsp3JFC/vokQ+dRrVI+hDBwz0SNmQ3pL6h39OSZlPs0uCG7wKJkftnaA== + dependencies: + "@types/node" "*" + "@types/ssh2-streams" "*" + "@types/stack-utils@^1.0.1": version "1.0.1" resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e" @@ -7683,7 +7698,7 @@ any-observable@^0.3.0: resolved "https://registry.npmjs.org/any-observable/-/any-observable-0.3.0.tgz#af933475e5806a67d0d7df090dd5e8bef65d119b" integrity sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog== -any-promise@^1.0.0: +any-promise@^1.0.0, any-promise@^1.1.0: version "1.3.0" resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" integrity sha1-q8av7tzqUugJzcA3au0845Y10X8= @@ -7960,6 +7975,19 @@ archiver@^5.0.2: tar-stream "^2.1.4" zip-stream "^4.0.4" +archiver@^5.3.0: + version "5.3.0" + resolved "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba" + integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg== + dependencies: + archiver-utils "^2.1.0" + async "^3.2.0" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.0.0" + tar-stream "^2.2.0" + zip-stream "^4.1.0" + are-we-there-yet@~1.1.2: version "1.1.5" resolved "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" @@ -10149,6 +10177,16 @@ compress-commons@^4.0.2: normalize-path "^3.0.0" readable-stream "^3.6.0" +compress-commons@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.0.tgz#25ec7a4528852ccd1d441a7d4353cd0ece11371b" + integrity sha512-ofaaLqfraD1YRTkrRKPCrGJ1pFeDG/MVCkVVV2FNGeWquSlqw5wOrwOfPQ1xF2u+blpeWASie5EubHz+vsNIgA== + dependencies: + buffer-crc32 "^0.2.13" + crc32-stream "^4.0.1" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + compressible@^2.0.12, compressible@~2.0.16: version "2.0.18" resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" @@ -11722,6 +11760,13 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" +docker-compose@^0.23.8: + version "0.23.10" + resolved "https://registry.npmjs.org/docker-compose/-/docker-compose-0.23.10.tgz#369fd2c6429754fb4134d3d29174a8c9569690e8" + integrity sha512-IzR6LzHrQyUvVwPNZY6F0oszAQLqHKOMNTN43Yu5aE6IBbhN9D/MpHbVUqHXTwzqIWiJM+ImYFjY5RdWWDGgfQ== + dependencies: + yaml "^1.10.2" + docker-modem@^2.1.0: version "2.1.3" resolved "https://registry.npmjs.org/docker-modem/-/docker-modem-2.1.3.tgz#15432225f63db02eb5de4bb9a621b7293e5f264d" @@ -14162,6 +14207,18 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.1.7: + version "7.1.7" + resolved "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" + integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + global-dirs@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/global-dirs/-/global-dirs-2.0.1.tgz#acdf3bb6685bcd55cb35e8a052266569e9469201" @@ -24266,6 +24323,14 @@ sqlstring@^2.3.2: resolved "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz#cdae7169389a1375b18e885f2e60b3e460809514" integrity sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg== +ssh-remote-port-forward@^1.0.3: + version "1.0.3" + resolved "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.3.tgz#074cebed6d54a42ce4659d6aa24987e433ff5fef" + integrity sha512-PJ6qGFmB6n0iMCQIp+qzx8qVUE5cgacopvEh63t3NJjEQHOaza/JT9zywmmVlaol/eGtCmpvBXx2A03ih1Y+xg== + dependencies: + "@types/ssh2" "^0.5.45" + ssh2 "^0.8.9" + ssh2-streams@~0.4.10: version "0.4.10" resolved "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34" @@ -24275,7 +24340,7 @@ ssh2-streams@~0.4.10: bcrypt-pbkdf "^1.0.2" streamsearch "~0.1.2" -ssh2@^0.8.7: +ssh2@^0.8.7, ssh2@^0.8.9: version "0.8.9" resolved "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3" integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw== @@ -24479,6 +24544,13 @@ stream-shift@^1.0.0: resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + integrity sha1-u/azn19D7DC8cbq8s3VXrOzzQ1M= + dependencies: + any-promise "^1.1.0" + stream-transform@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf" @@ -25033,7 +25105,7 @@ tapable@^1.0.0, tapable@^1.1.3: resolved "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== -tar-fs@^2.0.0: +tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -25064,6 +25136,17 @@ tar-stream@^2.0.0, tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +tar-stream@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@^2.0.0: version "2.2.2" resolved "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" @@ -25221,6 +25304,25 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" +testcontainers@^7.10.0: + version "7.11.0" + resolved "https://registry.npmjs.org/testcontainers/-/testcontainers-7.11.0.tgz#47291a7e693b6d8f7a4f03cd0fb2a6a741c53458" + integrity sha512-wnTS/foBu3lFjLHRCQLv+nSBnzgp20tWRJVCRXFzU6ivDnFwNbyLMZiHhHZr6hpXtCNAToOlBenlueDA5z1L7g== + dependencies: + "@types/archiver" "^5.1.0" + "@types/dockerode" "^3.2.1" + archiver "^5.3.0" + byline "^5.0.0" + debug "^4.3.1" + docker-compose "^0.23.8" + dockerode "^3.2.1" + get-port "^5.1.1" + glob "^7.1.7" + slash "^3.0.0" + ssh-remote-port-forward "^1.0.3" + stream-to-array "^2.3.0" + tar-fs "^2.1.1" + text-encoding-utf-8@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz#585b62197b0ae437e3c7b5d0af27ac1021e10d13" @@ -27168,7 +27270,7 @@ yaml-jest@^1.0.5: dependencies: js-yaml "^3.7.0" -yaml@^1.10.0, yaml@^1.7.2, yaml@^1.9.2: +yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2, yaml@^1.9.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== @@ -27350,6 +27452,15 @@ zip-stream@^4.0.4: compress-commons "^4.0.2" readable-stream "^3.6.0" +zip-stream@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79" + integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A== + dependencies: + archiver-utils "^2.1.0" + compress-commons "^4.1.0" + readable-stream "^3.6.0" + zombie@^6.1.4: version "6.1.4" resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47"