paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 48 Packages Projects Releases Wiki Activity

rename to backend-test-utils and rearrange a bit

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2021-05-26 18:04:28 +02:00
parent 8c0109abba
commit 330f4bacdb
24 changed files with 820 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wise-pens-count.md
-5
View File
@@ -1,5 +0,0 @@
---
'@backstage/plugin-catalog-backend': patch
---
Use `TestDatabases` from `@backstage/backend-testing`
.github/workflows/master-win.yml
+1 -39
View File
@@ -2,7 +2,6 @@ name: Master Build Windows
on:
workflow_dispatch:
pull_request:
push:
branches: [master]
@@ -14,41 +13,6 @@ jobs:
matrix:
node-version: [12.x, 14.x]
services:
postgres13:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432/tcp
postgres9:
image: postgres:9
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432/tcp
mysql8:
image: mysql:8
env:
MYSQL_ROOT_PASSWORD: root
options: >-
--health-cmd "mysqladmin ping -h localhost"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 3306/tcp
env:
CI: true
NODE_OPTIONS: --max-old-space-size=4096
@@ -92,9 +56,7 @@ jobs:
- name: test
run: yarn lerna -- run test
env:
BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }}
BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}
BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
BACKSTAGE_TEST_DISABLE_DOCKER: 1
# credit: https://github.com/appleboy/discord-action/issues/3#issuecomment-731426861
- name: Discord notification
.github/workflows/master.yml
+39
View File
@@ -16,6 +16,41 @@ jobs:
matrix:
node-version: [12.x, 14.x]
services:
postgres13:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432/tcp
postgres9:
image: postgres:9
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432/tcp
mysql8:
image: mysql:8
env:
MYSQL_ROOT_PASSWORD: root
options: >-
--health-cmd "mysqladmin ping -h localhost"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 3306/tcp
env:
CI: true
NODE_OPTIONS: --max-old-space-size=4096
@@ -86,6 +121,10 @@ jobs:
# Upload code coverage for some specific flags. Also see .codecov.yml
bash <(curl -s https://codecov.io/bash) -f packages/core/coverage/* -F core
bash <(curl -s https://codecov.io/bash) -f packages/core-api/coverage/* -F core-api
env:
BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }}
BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }}
BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING: mysql://root:root@localhost:${{ job.services.mysql8.ports[3306] }}/ignored
- name: Discord notification
if: ${{ failure() }}
packages/backend-testing/.eslintrc.js → packages/backend-test-utils/.eslintrc.js
View File
packages/backend-testing/README.md → packages/backend-test-utils/README.md
+2 -2
View File
@@ -1,4 +1,4 @@
# @backstage/backend-testing
# @backstage/backend-test-utils
Test helpers library for Backstage backends.
@@ -9,7 +9,7 @@ Add the library as a `devDependency` to your backend package:
```sh
# From the Backstage root directory, go to your backend package, or to a backend plugin
cd plugins/my-plugin-backend
yarn add --dev @backstage/backend-testing
yarn add --dev @backstage/backend-test-utils
```
## Documentation
packages/backend-test-utils/api-report.md
+31
View File
@@ -0,0 +1,31 @@
## API Report File for "@backstage/backend-test-utils"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Knex } from 'knex';
// @public (undocumented)
export function isDockerDisabledForTests(): boolean;
// @public
export type TestDatabaseId = 'POSTGRES_13' | 'POSTGRES_9' | 'MYSQL_8' | 'SQLITE_3';
// @public
export class TestDatabases {
static create(options?: {
ids?: TestDatabaseId[];
disableDocker?: boolean;
}): TestDatabases;
// (undocumented)
eachSupportedId(): [TestDatabaseId][];
init(id: TestDatabaseId): Promise<Knex>;
// (undocumented)
supports(id: TestDatabaseId): boolean;
}
// (No @packageDocumentation comment for this package)
```
packages/backend-testing/package.json → packages/backend-test-utils/package.json
+2 -2
View File
@@ -1,5 +1,5 @@
{
"name": "@backstage/backend-testing",
"name": "@backstage/backend-test-utils",
"description": "Test helpers library for Backstage backends",
"version": "0.1.1",
"main": "src/index.ts",
@@ -14,7 +14,7 @@
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "packages/backend-testing"
"directory": "packages/backend-test-utils"
},
"keywords": [
"backstage",
packages/backend-test-utils/src/database/TestDatabases.test.ts
+157
View File
@@ -0,0 +1,157 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knexFactory from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
import { startPostgresContainer } from './startPostgresContainer';
import { TestDatabases } from './TestDatabases';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
describe('TestDatabases', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterAll(() => {
process.env = OLD_ENV;
});
describe('each', () => {
const dbs = TestDatabases.create();
it.each(dbs.eachSupportedId())(
'creates distinct %p databases',
async databaseId => {
if (!dbs.supports(databaseId)) {
return;
}
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([
{ a: 1 },
]);
},
60_000,
);
});
itIfDocker(
'obeys a provided connection string for postgres 13',
async () => {
const dbs = TestDatabases.create();
const { host, port, user, password, stop } = await startPostgresContainer(
'postgres:13',
);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_13');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = 'backstage_plugin_0';
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
60_000,
);
itIfDocker(
'obeys a provided connection string for postgres 9',
async () => {
const dbs = TestDatabases.create();
const { host, port, user, password, stop } = await startPostgresContainer(
'postgres:9',
);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING = `postgresql://${user}:${password}@${host}:${port}`;
const input = await dbs.init('POSTGRES_9');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = 'backstage_plugin_0';
const output = knexFactory({
client: 'pg',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
60_000,
);
itIfDocker(
'obeys a provided connection string for mysql 8',
async () => {
const dbs = TestDatabases.create();
const { host, port, user, password, stop } = await startMysqlContainer(
'mysql:8',
);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING = `mysql://${user}:${password}@${host}:${port}/ignored`;
const input = await dbs.init('MYSQL_8');
await input.schema.createTable('a', table =>
table.string('x').primary(),
);
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const database = 'backstage_plugin_0';
const output = knexFactory({
client: 'mysql2',
connection: { host, port, user, password, database },
});
// eslint-disable-next-line jest/no-standalone-expect
await expect(output.select('x').from('a')).resolves.toEqual([
{ x: 'y' },
]);
} finally {
await stop();
}
},
60_000,
);
});
packages/backend-testing/src/database/TestDatabases.ts → packages/backend-test-utils/src/database/TestDatabases.ts
+97 -101
View File
@@ -17,66 +17,24 @@
import { SingleConnectionDatabaseManager } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { Knex } from 'knex';
import { GenericContainer, StartedTestContainer } from 'testcontainers';
import { v4 as uuid } from 'uuid';
type TestDatabaseProperties = {
name: string;
driver: string;
dockerImageName?: string;
connectionStringEnvironmentVariableName?: string;
};
type Instance = {
container?: StartedTestContainer;
databaseManager: SingleConnectionDatabaseManager;
connections: Array<Knex>;
};
const supportedDatabases = Object.freeze({
POSTGRES_13: {
name: 'Postgres 13.x',
driver: 'pg',
dockerImageName: 'postgres:13',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING',
},
POSTGRES_9: {
name: 'Postgres 9.x',
driver: 'pg',
dockerImageName: 'postgres:9',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING',
},
MYSQL_8: {
name: 'MySQL 8.x',
driver: 'mysql2',
dockerImageName: 'mysql:8',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING',
},
SQLITE_3: {
name: 'SQLite 3.x',
driver: 'sqlite3',
},
} as const);
/**
* The possible databases to test against.
*/
export type TestDatabaseId =
| 'POSTGRES_13'
| 'POSTGRES_9'
| 'MYSQL_8'
| 'SQLITE_3';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
import { startPostgresContainer } from './startPostgresContainer';
import {
allDatabases,
Instance,
TestDatabaseId,
TestDatabaseProperties,
} from './types';
/**
* Encapsulates the creation of ephemeral test database instances for use
* inside unit or integration tests.
*/
export class TestDatabases {
private instanceById: Map<string, Instance> = new Map();
private lastDatabaseId = 0;
private readonly instanceById: Map<string, Instance>;
private readonly supportedIds: TestDatabaseId[];
private lastDatabaseIndex: number;
/**
* Creates an empty `TestDatabases` instance, and sets up Jest to clean up
@@ -89,8 +47,43 @@ export class TestDatabases {
* But initializing a new logical database inside that instance using `init`
* is very fast.
*/
static create() {
const databases = new TestDatabases();
static create(options?: {
ids?: TestDatabaseId[];
disableDocker?: boolean;
}): TestDatabases {
const defaultOptions = {
ids: Object.keys(allDatabases) as TestDatabaseId[],
disableDocker: isDockerDisabledForTests(),
};
const { ids, disableDocker } = Object.assign(defaultOptions, options ?? {});
const supportedIds = ids.filter(id => {
const properties = allDatabases[id];
if (!properties) {
return false;
}
// If the caller has set up the env with an explicit connection string,
// we'll assume that this database will work
if (
properties.connectionStringEnvironmentVariableName &&
process.env[properties.connectionStringEnvironmentVariableName]
) {
return true;
}
// If the database doesn't require docker at all, there's nothing to worry
// about
if (!properties.dockerImageName) {
return true;
}
// If the database requires docker, but docker is disabled, we will fail.
if (disableDocker) {
return false;
}
return true;
});
const databases = new TestDatabases(supportedIds);
afterAll(async () => {
await databases.shutdown();
@@ -99,7 +92,19 @@ export class TestDatabases {
return databases;
}
private constructor() {}
private constructor(supportedIds: TestDatabaseId[]) {
this.instanceById = new Map();
this.supportedIds = supportedIds;
this.lastDatabaseIndex = 0;
}
supports(id: TestDatabaseId): boolean {
return this.supportedIds.includes(id);
}
eachSupportedId(): [TestDatabaseId][] {
return this.supportedIds.map(id => [id]);
}
/**
* Returns a fresh, unique, empty logical database on an instance of the
@@ -109,11 +114,17 @@ export class TestDatabases {
* @returns A `Knex` connection object
*/
async init(id: TestDatabaseId): Promise<Knex> {
const properties = supportedDatabases[id];
const properties = allDatabases[id];
if (!properties) {
const candidates = Object.keys(supportedDatabases).join(', ');
const candidates = Object.keys(allDatabases).join(', ');
throw new Error(
`Unsupported test database ${id}, possible values are ${candidates}`,
`Unknown test database ${id}, possible values are ${candidates}`,
);
}
if (!this.supportedIds.includes(id)) {
const candidates = this.supportedIds.join(', ');
throw new Error(
`Unsupported test database ${id} for this environment, possible values are ${candidates}`,
);
}
@@ -127,7 +138,7 @@ export class TestDatabases {
// Ensure that a unique logical database is created in the instance
const connection = await instance.databaseManager
.forPlugin(String(this.lastDatabaseId++))
.forPlugin(String(this.lastDatabaseIndex++))
.getClient();
instance.connections.push(connection);
@@ -176,32 +187,23 @@ export class TestDatabases {
private async initPostgres(
properties: TestDatabaseProperties,
): Promise<Instance> {
const password = uuid();
const container = await new GenericContainer(properties.dockerImageName!)
.withExposedPorts(5432)
.withEnv('POSTGRES_PASSWORD', password)
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const { host, port, user, password, stop } = await startPostgresContainer(
properties.dockerImageName!,
);
const databaseManager = SingleConnectionDatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'pg',
connection: {
host: container.getHost(),
port: container.getMappedPort(5432),
user: 'postgres',
password,
},
connection: { host, port, user, password },
},
},
}),
);
return {
container,
stopContainer: stop,
databaseManager,
connections: [],
};
@@ -210,32 +212,23 @@ export class TestDatabases {
private async initMysql(
properties: TestDatabaseProperties,
): Promise<Instance> {
const password = uuid();
const container = await new GenericContainer(properties.dockerImageName!)
.withExposedPorts(3306)
.withEnv('MYSQL_ROOT_PASSWORD', password)
.withTmpFs({ '/var/lib/mysql': 'rw' })
.start();
const { host, port, user, password, stop } = await startMysqlContainer(
properties.dockerImageName!,
);
const databaseManager = SingleConnectionDatabaseManager.fromConfig(
new ConfigReader({
backend: {
database: {
client: 'mysql2',
connection: {
host: container.getHost(),
port: container.getMappedPort(3306),
user: 'root',
password,
},
connection: { host, port, user, password },
},
},
}),
);
return {
container,
stopContainer: stop,
databaseManager,
connections: [],
};
@@ -262,17 +255,20 @@ export class TestDatabases {
}
private async shutdown() {
for (const { container, connections } of this.instanceById.values()) {
try {
await Promise.all(connections.map(c => c.destroy()));
} catch {
// ignore
}
try {
await container?.stop({ timeout: 10_000 });
} catch {
// ignore
}
}
const instances = [...this.instanceById.values()];
await Promise.all(
instances.map(async ({ stopContainer, connections }) => {
try {
await Promise.all(connections.map(c => c.destroy()));
} catch {
// ignore
}
try {
await stopContainer?.();
} catch {
// ignore
}
}),
);
}
}
packages/backend-test-utils/src/database/index.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
export { TestDatabases } from './TestDatabases';
export type { TestDatabaseId } from './types';
packages/backend-test-utils/src/database/startMysqlContainer.test.ts
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startMysqlContainer } from './startMysqlContainer';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
describe('startMysqlContainer', () => {
itIfDocker(
'successfully launches the container',
async () => {
const { stop, ...connection } = await startMysqlContainer('mysql:8');
const db = createConnection({ client: 'mysql2', connection });
try {
const result = await db.select(db.raw('version() AS version'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('8.');
} finally {
await db.destroy();
await stop();
}
},
60_000,
);
});
packages/backend-test-utils/src/database/startMysqlContainer.ts
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection, { Knex } from 'knex';
import { GenericContainer } from 'testcontainers';
import { v4 as uuid } from 'uuid';
async function waitForMysqlReady(
connection: Knex.MySqlConnectionConfig,
): Promise<void> {
const startTime = Date.now();
const db = createConnection({ client: 'mysql2', connection });
try {
for (;;) {
try {
const result = await db.select(db.raw('version() AS version'));
if (result[0]?.version) {
return;
}
} catch (e) {
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${e}`,
);
}
}
await new Promise(resolve => setTimeout(resolve, 100));
}
} finally {
db.destroy();
}
}
export async function startMysqlContainer(image: string) {
const user = 'root';
const password = uuid();
const container = await new GenericContainer(image)
.withExposedPorts(3306)
.withEnv('MYSQL_ROOT_PASSWORD', password)
.withTmpFs({ '/var/lib/mysql': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(3306);
const stop = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForMysqlReady({ host, port, user, password });
return { host, port, user, password, stop };
}
packages/backend-test-utils/src/database/startPostgresContainer.test.ts
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection from 'knex';
import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests';
import { startPostgresContainer } from './startPostgresContainer';
const itIfDocker = isDockerDisabledForTests() ? it.skip : it;
describe('startPostgresContainer', () => {
itIfDocker(
'successfully launches the container',
async () => {
const { stop, ...connection } = await startPostgresContainer(
'postgres:13',
);
const db = createConnection({ client: 'pg', connection });
try {
const result = await db.select(db.raw('version()'));
// eslint-disable-next-line jest/no-standalone-expect
expect(result[0]?.version).toContain('PostgreSQL');
} finally {
await db.destroy();
await stop();
}
},
60_000,
);
});
packages/backend-test-utils/src/database/startPostgresContainer.ts
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import createConnection, { Knex } from 'knex';
import { GenericContainer } from 'testcontainers';
import { v4 as uuid } from 'uuid';
async function waitForPostgresReady(
connection: Knex.PgConnectionConfig,
): Promise<void> {
const startTime = Date.now();
const db = createConnection({ client: 'pg', connection });
try {
for (;;) {
try {
const result = await db.select(db.raw('version()'));
if (Array.isArray(result) && result[0]?.version) {
return;
}
} catch (e) {
if (Date.now() - startTime > 30_000) {
throw new Error(
`Timed out waiting for the database to be ready for connections, ${e}`,
);
}
}
await new Promise(resolve => setTimeout(resolve, 100));
}
} finally {
db.destroy();
}
}
export async function startPostgresContainer(image: string) {
const user = 'postgres';
const password = uuid();
const container = await new GenericContainer(image)
.withExposedPorts(5432)
.withEnv('POSTGRES_PASSWORD', password)
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(5432);
const stop = async () => {
await container.stop({ timeout: 10_000 });
};
await waitForPostgresReady({ host, port, user, password });
return { host, port, user, password, stop };
}
packages/backend-test-utils/src/database/types.ts
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { SingleConnectionDatabaseManager } from '@backstage/backend-common';
import { Knex } from 'knex';
/**
* The possible databases to test against.
*/
export type TestDatabaseId =
| 'POSTGRES_13'
| 'POSTGRES_9'
| 'MYSQL_8'
| 'SQLITE_3';
export type TestDatabaseProperties = {
name: string;
driver: string;
dockerImageName?: string;
connectionStringEnvironmentVariableName?: string;
};
export type Instance = {
stopContainer?: () => Promise<void>;
databaseManager: SingleConnectionDatabaseManager;
connections: Array<Knex>;
};
export const allDatabases: Record<
TestDatabaseId,
TestDatabaseProperties
> = Object.freeze({
POSTGRES_13: {
name: 'Postgres 13.x',
driver: 'pg',
dockerImageName: 'postgres:13',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING',
},
POSTGRES_9: {
name: 'Postgres 9.x',
driver: 'pg',
dockerImageName: 'postgres:9',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING',
},
MYSQL_8: {
name: 'MySQL 8.x',
driver: 'mysql2',
dockerImageName: 'mysql:8',
connectionStringEnvironmentVariableName:
'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING',
},
SQLITE_3: {
name: 'SQLite 3.x',
driver: 'sqlite3',
},
});
packages/backend-testing/src/index.ts → packages/backend-test-utils/src/index.ts
+1
View File
@@ -15,3 +15,4 @@
*/
export * from './database';
export * from './util';
packages/backend-testing/src/setupTests.ts → packages/backend-test-utils/src/setupTests.ts
View File
packages/backend-testing/src/database/index.ts → packages/backend-test-utils/src/util/index.ts
+1 -2
View File
@@ -14,5 +14,4 @@
* limitations under the License.
*/
export { TestDatabases } from './TestDatabases';
export type { TestDatabaseId } from './TestDatabases';
export { isDockerDisabledForTests } from './isDockerDisabledForTests';
packages/backend-test-utils/src/util/isDockerDisabledForTests.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function isDockerDisabledForTests() {
return Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER);
}
packages/backend-testing/api-report.md
-21
View File
@@ -1,21 +0,0 @@
## API Report File for "@backstage/backend-testing"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Knex } from 'knex';
// @public
export type TestDatabaseId = 'POSTGRES_13' | 'POSTGRES_9' | 'MYSQL_8' | 'SQLITE_3';
// @public
export class TestDatabases {
static create(): TestDatabases;
init(id: TestDatabaseId): Promise<Knex>;
}
// (No @packageDocumentation comment for this package)
```
packages/backend-testing/src/database/TestDatabases.test.ts
-154
View File
@@ -1,154 +0,0 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import knexFactory from 'knex';
import { GenericContainer } from 'testcontainers';
import { TestDatabases } from './TestDatabases';
describe('TestDatabases', () => {
const OLD_ENV = process.env;
beforeEach(() => {
jest.resetModules();
process.env = { ...OLD_ENV };
});
afterAll(() => {
process.env = OLD_ENV;
});
it.each([
['POSTGRES_13'],
['POSTGRES_9'],
['MYSQL_8'],
['SQLITE_3'],
] as const)(
'creates distinct %p databases',
async databaseId => {
const dbs = TestDatabases.create();
const db1 = await dbs.init(databaseId);
const db2 = await dbs.init(databaseId);
await db1.schema.createTable('a', table => table.string('x').primary());
await db2.schema.createTable('a', table => table.string('y').primary());
await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([
{ a: 1 },
]);
},
60_000,
);
it('obeys a provided connection string for postgres 13', async () => {
const dbs = TestDatabases.create();
const container = await new GenericContainer('postgres:13')
.withExposedPorts(5432)
.withEnv('POSTGRES_USER', 'user')
.withEnv('POSTGRES_PASSWORD', 'pass')
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(5432);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING = `postgresql://user:pass@${host}:${port}`;
const input = await dbs.init('POSTGRES_13');
await input.schema.createTable('a', table => table.string('x').primary());
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const output = knexFactory({
client: 'pg',
connection: {
host,
port,
user: 'user',
password: 'pass',
database: 'backstage_plugin_0',
},
});
await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]);
} finally {
await container.stop({ timeout: 10_000 });
}
}, 60_000);
it('obeys a provided connection string for postgres 9', async () => {
const dbs = TestDatabases.create();
const container = await new GenericContainer('postgres:9')
.withExposedPorts(5432)
.withEnv('POSTGRES_USER', 'user')
.withEnv('POSTGRES_PASSWORD', 'pass')
.withTmpFs({ '/var/lib/postgresql/data': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(5432);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING = `postgresql://user:pass@${host}:${port}`;
const input = await dbs.init('POSTGRES_9');
await input.schema.createTable('a', table => table.string('x').primary());
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const output = knexFactory({
client: 'pg',
connection: {
host,
port,
user: 'user',
password: 'pass',
database: 'backstage_plugin_0',
},
});
await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]);
} finally {
await container.stop({ timeout: 10_000 });
}
}, 60_000);
it('obeys a provided connection string for mysql 8', async () => {
const dbs = TestDatabases.create();
const container = await new GenericContainer('mysql:8')
.withExposedPorts(3306)
.withEnv('MYSQL_ROOT_PASSWORD', 'pass')
.withTmpFs({ '/var/lib/mysql': 'rw' })
.start();
const host = container.getHost();
const port = container.getMappedPort(3306);
try {
// Leave a mark
process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING = `mysql://root:pass@${host}:${port}/ignored`;
const input = await dbs.init('MYSQL_8');
await input.schema.createTable('a', table => table.string('x').primary());
await input.insert({ x: 'y' }).into('a');
// Look for the mark
const output = knexFactory({
client: 'mysql2',
connection: {
host,
port,
user: 'root',
password: 'pass',
database: 'backstage_plugin_0',
},
});
await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]);
} finally {
await container.stop({ timeout: 10_000 });
}
}, 60_000);
});
plugins/catalog-backend/package.json
-1
View File
@@ -64,7 +64,6 @@
"yup": "^0.29.3"
},
"devDependencies": {
"@backstage/backend-testing": "^0.1.0",
"@backstage/cli": "^0.6.12",
"@backstage/test-utils": "^0.1.12",
"@types/core-js": "^2.5.4",
plugins/catalog-backend/src/next/Stitcher.test.ts
+161 -180
View File
@@ -15,7 +15,7 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TestDatabases } from '@backstage/backend-testing';
import { Knex } from 'knex';
import { DatabaseManager } from './database/DatabaseManager';
import {
DbRefreshStateReferencesRow,
@@ -26,200 +26,181 @@ import { DbSearchRow } from './search';
import { DbFinalEntitiesRow, Stitcher } from './Stitcher';
describe('Stitcher', () => {
const databases = TestDatabases.create();
let db: Knex;
const logger = getVoidLogger();
it.each([
['POSTGRES_13'],
['POSTGRES_9'],
['SQLITE_3'],
// TODO(freben): mysql:8 is not ready to use yet
] as const)(
'runs the happy path for %p',
async databaseId => {
const db = await databases.init(databaseId);
await DatabaseManager.createDatabase(db);
beforeEach(async () => {
db = await DatabaseManager.createTestDatabaseConnection();
await DatabaseManager.createDatabase(db);
});
const stitcher = new Stitcher(db, logger);
it('runs the happy path', async () => {
const stitcher = new Stitcher(db, logger);
await db.transaction(async tx => {
await tx<DbRefreshStateRow>('refresh_state').insert([
await db.transaction(async tx => {
await tx<DbRefreshStateRow>('refresh_state').insert([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: tx.fn.now(),
last_discovery_at: tx.fn.now(),
},
]);
await tx<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await tx<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
]);
});
await stitcher.stitch(new Set(['k:ns/n']));
let firstHash: string;
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: [
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
type: 'looksAt',
target: {
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: tx.fn.now(),
last_discovery_at: tx.fn.now(),
namespace: 'ns',
name: 'other',
},
},
]);
await tx<DbRefreshStateReferencesRow>(
'refresh_state_references',
).insert([{ source_key: 'a', target_entity_ref: 'k:ns/n' }]);
await tx<DbRelationsRow>('relations').insert([
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
firstHash = entities[0].hash;
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
});
// Re-stitch without any changes
await stitcher.stitch(new Set(['k:ns/n']));
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
});
// Now add one more relation and re-stitch
await db.transaction(async tx => {
await tx<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/third',
},
]);
});
await stitcher.stitch(new Set(['k:ns/n']));
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
]);
});
await stitcher.stitch(new Set(['k:ns/n']));
let firstHash: string;
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
firstHash = entities[0].hash;
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
value: 'k:ns/other',
},
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
});
// Re-stitch without any changes
await stitcher.stitch(new Set(['k:ns/n']));
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
});
// Now add one more relation and re-stitch
await db.transaction(async tx => {
await tx<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/third',
target: {
kind: 'k',
namespace: 'ns',
name: 'third',
},
},
]);
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
await stitcher.stitch(new Set(['k:ns/n']));
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
},
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'third',
},
},
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{
entity_id: 'my-id',
key: 'relations.looksat',
value: 'k:ns/other',
},
{
entity_id: 'my-id',
key: 'relations.looksat',
value: 'k:ns/third',
},
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
});
},
30000,
);
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
});
});
});
scripts/api-extractor.ts
+1 -1
View File
@@ -62,7 +62,7 @@ PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = function tryGetPackag
const DOCUMENTED_PACKAGES = [
'packages/backend-common',
'packages/backend-testing',
'packages/backend-test-utils',
'packages/catalog-client',
'packages/catalog-model',
'packages/cli-common',