From babe473dd10f9ef59c734a0f301f2ba3762c41e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 10 May 2021 10:31:50 +0200 Subject: [PATCH] Introduce `@backstage/backend-testing`, with `TestDatabases` to start off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/wise-pens-count.md | 5 + packages/backend-testing/.eslintrc.js | 3 + packages/backend-testing/README.md | 18 + packages/backend-testing/api-report.md | 21 ++ packages/backend-testing/package.json | 50 +++ .../src/database/TestDatabases.test.ts | 154 ++++++++ .../src/database/TestDatabases.ts | 278 ++++++++++++++ .../backend-testing/src/database/index.ts | 18 + packages/backend-testing/src/index.ts | 17 + packages/backend-testing/src/setupTests.ts | 17 + plugins/catalog-backend/package.json | 1 + .../catalog-backend/src/next/Stitcher.test.ts | 341 +++++++++--------- scripts/api-extractor.ts | 1 + yarn.lock | 107 +++++- 14 files changed, 866 insertions(+), 165 deletions(-) create mode 100644 .changeset/wise-pens-count.md create mode 100644 packages/backend-testing/.eslintrc.js create mode 100644 packages/backend-testing/README.md create mode 100644 packages/backend-testing/api-report.md create mode 100644 packages/backend-testing/package.json create mode 100644 packages/backend-testing/src/database/TestDatabases.test.ts create mode 100644 packages/backend-testing/src/database/TestDatabases.ts create mode 100644 packages/backend-testing/src/database/index.ts create mode 100644 packages/backend-testing/src/index.ts create mode 100644 packages/backend-testing/src/setupTests.ts diff --git a/.changeset/wise-pens-count.md b/.changeset/wise-pens-count.md new file mode 100644 index 0000000000..26eda866bc --- /dev/null +++ b/.changeset/wise-pens-count.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Use `TestDatabases` from `@backstage/backend-testing` diff --git a/packages/backend-testing/.eslintrc.js b/packages/backend-testing/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/backend-testing/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/backend-testing/README.md b/packages/backend-testing/README.md new file mode 100644 index 0000000000..f12e94ac25 --- /dev/null +++ b/packages/backend-testing/README.md @@ -0,0 +1,18 @@ +# @backstage/backend-testing + +Test helpers library for Backstage backends. + +## Usage + +Add the library as a `devDependency` to your backend package: + +```sh +# From the Backstage root directory, go to your backend package, or to a backend plugin +cd plugins/my-plugin-backend +yarn add --dev @backstage/backend-testing +``` + +## Documentation + +- [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) +- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md) diff --git a/packages/backend-testing/api-report.md b/packages/backend-testing/api-report.md new file mode 100644 index 0000000000..c4c78408ec --- /dev/null +++ b/packages/backend-testing/api-report.md @@ -0,0 +1,21 @@ +## API Report File for "@backstage/backend-testing" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { Knex } from 'knex'; + +// @public +export type TestDatabaseId = 'POSTGRES_13' | 'POSTGRES_9' | 'MYSQL_8' | 'SQLITE_3'; + +// @public +export class TestDatabases { + static create(): TestDatabases; + init(id: TestDatabaseId): Promise; + } + + +// (No @packageDocumentation comment for this package) + +``` diff --git a/packages/backend-testing/package.json b/packages/backend-testing/package.json new file mode 100644 index 0000000000..345d294a6c --- /dev/null +++ b/packages/backend-testing/package.json @@ -0,0 +1,50 @@ +{ + "name": "@backstage/backend-testing", + "description": "Test helpers library for Backstage backends", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/backend-testing" + }, + "keywords": [ + "backstage", + "test" + ], + "license": "Apache-2.0", + "scripts": { + "build": "backstage-cli build --outputs cjs,types", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.7.0", + "@backstage/cli": "^0.6.10", + "@backstage/config": "^0.1.5", + "knex": "^0.95.1", + "mysql2": "^2.2.5", + "pg": "^8.3.0", + "sqlite3": "^5.0.0", + "testcontainers": "^7.10.0", + "uuid": "^8.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.6.10", + "jest": "^26.0.1" + }, + "files": [ + "dist" + ] +} diff --git a/packages/backend-testing/src/database/TestDatabases.test.ts b/packages/backend-testing/src/database/TestDatabases.test.ts new file mode 100644 index 0000000000..61220e3090 --- /dev/null +++ b/packages/backend-testing/src/database/TestDatabases.test.ts @@ -0,0 +1,154 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import knexFactory from 'knex'; +import { GenericContainer } from 'testcontainers'; +import { TestDatabases } from './TestDatabases'; + +describe('TestDatabases', () => { + const OLD_ENV = process.env; + beforeEach(() => { + jest.resetModules(); + process.env = { ...OLD_ENV }; + }); + afterAll(() => { + process.env = OLD_ENV; + }); + + it.each([ + ['POSTGRES_13'], + ['POSTGRES_9'], + ['MYSQL_8'], + ['SQLITE_3'], + ] as const)( + 'creates distinct %p databases', + async databaseId => { + const dbs = TestDatabases.create(); + const db1 = await dbs.init(databaseId); + const db2 = await dbs.init(databaseId); + await db1.schema.createTable('a', table => table.string('x').primary()); + await db2.schema.createTable('a', table => table.string('y').primary()); + await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([ + { a: 1 }, + ]); + }, + 60_000, + ); + + it('obeys a provided connection string for postgres 13', async () => { + const dbs = TestDatabases.create(); + const container = await new GenericContainer('postgres:13') + .withExposedPorts(5432) + .withEnv('POSTGRES_USER', 'user') + .withEnv('POSTGRES_PASSWORD', 'pass') + .withTmpFs({ '/var/lib/postgresql/data': 'rw' }) + .start(); + const host = container.getHost(); + const port = container.getMappedPort(5432); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING = `postgresql://user:pass@${host}:${port}`; + const input = await dbs.init('POSTGRES_13'); + await input.schema.createTable('a', table => table.string('x').primary()); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const output = knexFactory({ + client: 'pg', + connection: { + host, + port, + user: 'user', + password: 'pass', + database: 'backstage_plugin_0', + }, + }); + await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]); + } finally { + await container.stop({ timeout: 10_000 }); + } + }, 60_000); + + it('obeys a provided connection string for postgres 9', async () => { + const dbs = TestDatabases.create(); + const container = await new GenericContainer('postgres:9') + .withExposedPorts(5432) + .withEnv('POSTGRES_USER', 'user') + .withEnv('POSTGRES_PASSWORD', 'pass') + .withTmpFs({ '/var/lib/postgresql/data': 'rw' }) + .start(); + const host = container.getHost(); + const port = container.getMappedPort(5432); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING = `postgresql://user:pass@${host}:${port}`; + const input = await dbs.init('POSTGRES_9'); + await input.schema.createTable('a', table => table.string('x').primary()); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const output = knexFactory({ + client: 'pg', + connection: { + host, + port, + user: 'user', + password: 'pass', + database: 'backstage_plugin_0', + }, + }); + await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]); + } finally { + await container.stop({ timeout: 10_000 }); + } + }, 60_000); + + it('obeys a provided connection string for mysql 8', async () => { + const dbs = TestDatabases.create(); + const container = await new GenericContainer('mysql:8') + .withExposedPorts(3306) + .withEnv('MYSQL_ROOT_PASSWORD', 'pass') + .withTmpFs({ '/var/lib/mysql': 'rw' }) + .start(); + const host = container.getHost(); + const port = container.getMappedPort(3306); + + try { + // Leave a mark + process.env.BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING = `mysql://root:pass@${host}:${port}/ignored`; + const input = await dbs.init('MYSQL_8'); + await input.schema.createTable('a', table => table.string('x').primary()); + await input.insert({ x: 'y' }).into('a'); + + // Look for the mark + const output = knexFactory({ + client: 'mysql2', + connection: { + host, + port, + user: 'root', + password: 'pass', + database: 'backstage_plugin_0', + }, + }); + await expect(output.select('x').from('a')).resolves.toEqual([{ x: 'y' }]); + } finally { + await container.stop({ timeout: 10_000 }); + } + }, 60_000); +}); diff --git a/packages/backend-testing/src/database/TestDatabases.ts b/packages/backend-testing/src/database/TestDatabases.ts new file mode 100644 index 0000000000..927ff2632c --- /dev/null +++ b/packages/backend-testing/src/database/TestDatabases.ts @@ -0,0 +1,278 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SingleConnectionDatabaseManager } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { Knex } from 'knex'; +import { GenericContainer, StartedTestContainer } from 'testcontainers'; +import { v4 as uuid } from 'uuid'; + +type TestDatabaseProperties = { + name: string; + driver: string; + dockerImageName?: string; + connectionStringEnvironmentVariableName?: string; +}; + +type Instance = { + container?: StartedTestContainer; + databaseManager: SingleConnectionDatabaseManager; + connections: Array; +}; + +const supportedDatabases = Object.freeze({ + POSTGRES_13: { + name: 'Postgres 13.x', + driver: 'pg', + dockerImageName: 'postgres:13', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING', + }, + POSTGRES_9: { + name: 'Postgres 9.x', + driver: 'pg', + dockerImageName: 'postgres:9', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING', + }, + MYSQL_8: { + name: 'MySQL 8.x', + driver: 'mysql2', + dockerImageName: 'mysql:8', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING', + }, + SQLITE_3: { + name: 'SQLite 3.x', + driver: 'sqlite3', + }, +} as const); + +/** + * The possible databases to test against. + */ +export type TestDatabaseId = + | 'POSTGRES_13' + | 'POSTGRES_9' + | 'MYSQL_8' + | 'SQLITE_3'; + +/** + * Encapsulates the creation of ephemeral test database instances for use + * inside unit or integration tests. + */ +export class TestDatabases { + private instanceById: Map = new Map(); + private lastDatabaseId = 0; + + /** + * Creates an empty `TestDatabases` instance, and sets up Jest to clean up + * all of its acquired resources after all tests finish. + * + * You typically want to create just a single instance like this at the top + * of your test file or `describe` block, and then call `init` many times on + * that instance inside the individual tests. Spinning up a "physical" + * database instance takes a considerable amount of time, slowing down tests. + * But initializing a new logical database inside that instance using `init` + * is very fast. + */ + static create() { + const databases = new TestDatabases(); + + afterAll(async () => { + await databases.shutdown(); + }); + + return databases; + } + + private constructor() {} + + /** + * Returns a fresh, unique, empty logical database on an instance of the + * given database ID platform. + * + * @param id The ID of the database platform to use, e.g. 'POSTGRES_13' + * @returns A `Knex` connection object + */ + async init(id: TestDatabaseId): Promise { + const properties = supportedDatabases[id]; + if (!properties) { + const candidates = Object.keys(supportedDatabases).join(', '); + throw new Error( + `Unsupported test database ${id}, possible values are ${candidates}`, + ); + } + + let instance: Instance | undefined = this.instanceById.get(id); + + // Ensure that a testcontainers instance is up for this ID + if (!instance) { + instance = await this.initAny(properties); + this.instanceById.set(id, instance); + } + + // Ensure that a unique logical database is created in the instance + const connection = await instance.databaseManager + .forPlugin(String(this.lastDatabaseId++)) + .getClient(); + + instance.connections.push(connection); + + return connection; + } + + private async initAny(properties: TestDatabaseProperties): Promise { + // Use the connection string if provided + if (properties.driver === 'pg' || properties.driver === 'mysql2') { + const envVarName = properties.connectionStringEnvironmentVariableName; + if (envVarName) { + const connectionString = process.env[envVarName]; + if (connectionString) { + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: properties.driver, + connection: connectionString, + }, + }, + }), + ); + return { + databaseManager, + connections: [], + }; + } + } + } + + // Otherwise start a container for the purpose + switch (properties.driver) { + case 'pg': + return this.initPostgres(properties); + case 'mysql2': + return this.initMysql(properties); + case 'sqlite3': + return this.initSqlite(properties); + default: + throw new Error(`Unknown database driver ${properties.driver}`); + } + } + + private async initPostgres( + properties: TestDatabaseProperties, + ): Promise { + const password = uuid(); + + const container = await new GenericContainer(properties.dockerImageName!) + .withExposedPorts(5432) + .withEnv('POSTGRES_PASSWORD', password) + .withTmpFs({ '/var/lib/postgresql/data': 'rw' }) + .start(); + + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'pg', + connection: { + host: container.getHost(), + port: container.getMappedPort(5432), + user: 'postgres', + password, + }, + }, + }, + }), + ); + + return { + container, + databaseManager, + connections: [], + }; + } + + private async initMysql( + properties: TestDatabaseProperties, + ): Promise { + const password = uuid(); + + const container = await new GenericContainer(properties.dockerImageName!) + .withExposedPorts(3306) + .withEnv('MYSQL_ROOT_PASSWORD', password) + .withTmpFs({ '/var/lib/mysql': 'rw' }) + .start(); + + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'mysql2', + connection: { + host: container.getHost(), + port: container.getMappedPort(3306), + user: 'root', + password, + }, + }, + }, + }), + ); + + return { + container, + databaseManager, + connections: [], + }; + } + + private async initSqlite( + _properties: TestDatabaseProperties, + ): Promise { + const databaseManager = SingleConnectionDatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { + client: 'sqlite3', + connection: ':memory:', + }, + }, + }), + ); + + return { + databaseManager, + connections: [], + }; + } + + private async shutdown() { + for (const { container, connections } of this.instanceById.values()) { + try { + await Promise.all(connections.map(c => c.destroy())); + } catch { + // ignore + } + try { + await container?.stop({ timeout: 10_000 }); + } catch { + // ignore + } + } + } +} diff --git a/packages/backend-testing/src/database/index.ts b/packages/backend-testing/src/database/index.ts new file mode 100644 index 0000000000..b88ba38010 --- /dev/null +++ b/packages/backend-testing/src/database/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { TestDatabases } from './TestDatabases'; +export type { TestDatabaseId } from './TestDatabases'; diff --git a/packages/backend-testing/src/index.ts b/packages/backend-testing/src/index.ts new file mode 100644 index 0000000000..3febc35a77 --- /dev/null +++ b/packages/backend-testing/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './database'; diff --git a/packages/backend-testing/src/setupTests.ts b/packages/backend-testing/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/packages/backend-testing/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index c8e17bb943..e6fa30882c 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -63,6 +63,7 @@ "yup": "^0.29.3" }, "devDependencies": { + "@backstage/backend-testing": "^0.1.0", "@backstage/cli": "^0.6.10", "@backstage/test-utils": "^0.1.9", "@types/core-js": "^2.5.4", diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts index f38af384fa..e4ff41039d 100644 --- a/plugins/catalog-backend/src/next/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -15,7 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-testing'; import { DatabaseManager } from './database/DatabaseManager'; import { DbRefreshStateReferencesRow, @@ -26,181 +26,200 @@ import { DbSearchRow } from './search'; import { DbFinalEntitiesRow, Stitcher } from './Stitcher'; describe('Stitcher', () => { - let db: Knex; + const databases = TestDatabases.create(); const logger = getVoidLogger(); - beforeEach(async () => { - db = await DatabaseManager.createTestDatabaseConnection(); - await DatabaseManager.createDatabase(db); - }); + it.each([ + ['POSTGRES_13'], + ['POSTGRES_9'], + ['SQLITE_3'], + // TODO(freben): mysql:8 is not ready to use yet + ] as const)( + 'runs the happy path for %p', + async databaseId => { + const db = await databases.init(databaseId); + await DatabaseManager.createDatabase(db); - it('runs the happy path', async () => { - const stitcher = new Stitcher(db, logger); + const stitcher = new Stitcher(db, logger); - await db.transaction(async tx => { - await tx('refresh_state').insert([ - { - entity_id: 'my-id', - entity_ref: 'k:ns/n', - unprocessed_entity: JSON.stringify({}), - processed_entity: JSON.stringify({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - }, - spec: { - k: 'v', - }, - }), - errors: '[]', - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - }, - ]); - await tx('refresh_state_references').insert([ - { source_key: 'a', target_entity_ref: 'k:ns/n' }, - ]); - await tx('relations').insert([ - { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/other', - }, - ]); - }); - - await stitcher.stitch(new Set(['k:ns/n'])); - - let firstHash: string; - await db.transaction(async tx => { - const entities = await tx('final_entities'); - - expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].final_entity); - expect(entity).toEqual({ - relations: [ + await db.transaction(async tx => { + await tx('refresh_state').insert([ { - type: 'looksAt', - target: { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', kind: 'k', - namespace: 'ns', - name: 'other', - }, + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), }, - ], - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - generation: 1, - uid: 'my-id', - }, - spec: { - k: 'v', - }, + ]); + await tx( + 'refresh_state_references', + ).insert([{ source_key: 'a', target_entity_ref: 'k:ns/n' }]); + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/other', + }, + ]); }); - expect(entity.metadata.etag).toEqual(entities[0].hash); - firstHash = entities[0].hash; + await stitcher.stitch(new Set(['k:ns/n'])); - const search = await tx('search'); - expect(search).toEqual( - expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, - ]), - ); - }); + let firstHash: string; + await db.transaction(async tx => { + const entities = await tx('final_entities'); - // Re-stitch without any changes - await stitcher.stitch(new Set(['k:ns/n'])); - - await db.transaction(async tx => { - const entities = await tx('final_entities'); - expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].final_entity); - expect(entities[0].hash).toEqual(firstHash); - expect(entity.metadata.etag).toEqual(firstHash); - }); - - // Now add one more relation and re-stitch - await db.transaction(async tx => { - await tx('relations').insert([ - { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/third', - }, - ]); - }); - - await stitcher.stitch(new Set(['k:ns/n'])); - - await db.transaction(async tx => { - const entities = await tx('final_entities'); - - expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].final_entity); - expect(entity).toEqual({ - relations: expect.arrayContaining([ - { - type: 'looksAt', - target: { - kind: 'k', - namespace: 'ns', - name: 'other', + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entity).toEqual({ + relations: [ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, }, + ], + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', }, - { - type: 'looksAt', - target: { - kind: 'k', - namespace: 'ns', - name: 'third', + spec: { + k: 'v', + }, + }); + + expect(entity.metadata.etag).toEqual(entities[0].hash); + firstHash = entities[0].hash; + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + key: 'relations.looksat', + value: 'k:ns/other', }, - }, - ]), - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - generation: 1, - uid: 'my-id', - }, - spec: { - k: 'v', - }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); }); - expect(entities[0].hash).not.toEqual(firstHash); - expect(entities[0].hash).toEqual(entity.metadata.etag); + // Re-stitch without any changes + await stitcher.stitch(new Set(['k:ns/n'])); - const search = await tx('search'); - expect(search).toEqual( - expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, - ]), - ); - }); - }); + await db.transaction(async tx => { + const entities = await tx('final_entities'); + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entities[0].hash).toEqual(firstHash); + expect(entity.metadata.etag).toEqual(firstHash); + }); + + // Now add one more relation and re-stitch + await db.transaction(async tx => { + await tx('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/third', + }, + ]); + }); + + await stitcher.stitch(new Set(['k:ns/n'])); + + await db.transaction(async tx => { + const entities = await tx('final_entities'); + + expect(entities.length).toBe(1); + const entity = JSON.parse(entities[0].final_entity); + expect(entity).toEqual({ + relations: expect.arrayContaining([ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'third', + }, + }, + ]), + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entities[0].hash).not.toEqual(firstHash); + expect(entities[0].hash).toEqual(entity.metadata.etag); + + const search = await tx('search'); + expect(search).toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + key: 'relations.looksat', + value: 'k:ns/other', + }, + { + entity_id: 'my-id', + key: 'relations.looksat', + value: 'k:ns/third', + }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }); + }, + 30000, + ); }); diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts index 37dc4769fd..4fe76cb38d 100644 --- a/scripts/api-extractor.ts +++ b/scripts/api-extractor.ts @@ -62,6 +62,7 @@ PackageJsonLookup.prototype.tryGetPackageJsonFilePathFor = function tryGetPackag const DOCUMENTED_PACKAGES = [ 'packages/backend-common', + 'packages/backend-testing', 'packages/catalog-client', 'packages/catalog-model', 'packages/cli-common', diff --git a/yarn.lock b/yarn.lock index a21c6ef0f1..ae7b15a344 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6763,6 +6763,21 @@ resolved "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9" integrity sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA== +"@types/ssh2-streams@*": + version "0.1.8" + resolved "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.8.tgz#142af404dae059931aea7fcd1511b5478964feb6" + integrity sha512-I7gixRPUvVIyJuCEvnmhr3KvA2dC0639kKswqD4H5b4/FOcnPtNU+qWLiXdKIqqX9twUvi5j0U1mwKE5CUsrfA== + dependencies: + "@types/node" "*" + +"@types/ssh2@^0.5.45": + version "0.5.46" + resolved "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.46.tgz#e12341a242aea0e98ac2dec89e039bf421fd3584" + integrity sha512-1pC8FHrMPYdkLoUOwTYYifnSEPzAFZRsp3JFC/vokQ+dRrVI+hDBwz0SNmQ3pL6h39OSZlPs0uCG7wKJkftnaA== + dependencies: + "@types/node" "*" + "@types/ssh2-streams" "*" + "@types/stack-utils@^1.0.1": version "1.0.1" resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e" @@ -7606,7 +7621,7 @@ any-observable@^0.3.0: resolved "https://registry.npmjs.org/any-observable/-/any-observable-0.3.0.tgz#af933475e5806a67d0d7df090dd5e8bef65d119b" integrity sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog== -any-promise@^1.0.0: +any-promise@^1.0.0, any-promise@^1.1.0: version "1.3.0" resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" integrity sha1-q8av7tzqUugJzcA3au0845Y10X8= @@ -7883,6 +7898,19 @@ archiver@^5.0.2: tar-stream "^2.1.4" zip-stream "^4.0.4" +archiver@^5.2.0: + version "5.3.0" + resolved "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba" + integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg== + dependencies: + archiver-utils "^2.1.0" + async "^3.2.0" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.0.0" + tar-stream "^2.2.0" + zip-stream "^4.1.0" + are-we-there-yet@~1.1.2: version "1.1.5" resolved "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" @@ -10118,6 +10146,16 @@ compress-commons@^4.0.2: normalize-path "^3.0.0" readable-stream "^3.6.0" +compress-commons@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.0.tgz#25ec7a4528852ccd1d441a7d4353cd0ece11371b" + integrity sha512-ofaaLqfraD1YRTkrRKPCrGJ1pFeDG/MVCkVVV2FNGeWquSlqw5wOrwOfPQ1xF2u+blpeWASie5EubHz+vsNIgA== + dependencies: + buffer-crc32 "^0.2.13" + crc32-stream "^4.0.1" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + compressible@^2.0.12, compressible@~2.0.16: version "2.0.18" resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" @@ -11675,6 +11713,13 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" +docker-compose@^0.23.5: + version "0.23.9" + resolved "https://registry.npmjs.org/docker-compose/-/docker-compose-0.23.9.tgz#d24343397792562f72a53eb895e76f5c7c751ee8" + integrity sha512-ucDcqBoavyQMuxPOpc8y0LqtALc5Y7h/Ga/FEiZ8m+ZqaVf3WuCh5HODtqvGO8HZMGrcH7HPyBqJ+gRJG+dBvg== + dependencies: + yaml "^1.10.2" + docker-modem@^2.1.0: version "2.1.3" resolved "https://registry.npmjs.org/docker-modem/-/docker-modem-2.1.3.tgz#15432225f63db02eb5de4bb9a621b7293e5f264d" @@ -24184,6 +24229,14 @@ sqlstring@^2.3.2: resolved "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz#cdae7169389a1375b18e885f2e60b3e460809514" integrity sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg== +ssh-remote-port-forward@^1.0.3: + version "1.0.3" + resolved "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.3.tgz#074cebed6d54a42ce4659d6aa24987e433ff5fef" + integrity sha512-PJ6qGFmB6n0iMCQIp+qzx8qVUE5cgacopvEh63t3NJjEQHOaza/JT9zywmmVlaol/eGtCmpvBXx2A03ih1Y+xg== + dependencies: + "@types/ssh2" "^0.5.45" + ssh2 "^0.8.9" + ssh2-streams@~0.4.10: version "0.4.10" resolved "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34" @@ -24193,7 +24246,7 @@ ssh2-streams@~0.4.10: bcrypt-pbkdf "^1.0.2" streamsearch "~0.1.2" -ssh2@^0.8.7: +ssh2@^0.8.7, ssh2@^0.8.9: version "0.8.9" resolved "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3" integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw== @@ -24397,6 +24450,13 @@ stream-shift@^1.0.0: resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + integrity sha1-u/azn19D7DC8cbq8s3VXrOzzQ1M= + dependencies: + any-promise "^1.1.0" + stream-transform@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf" @@ -24951,7 +25011,7 @@ tapable@^1.0.0, tapable@^1.1.3: resolved "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== -tar-fs@^2.0.0: +tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== @@ -24982,6 +25042,17 @@ tar-stream@^2.0.0, tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +tar-stream@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@^2.0.0: version "2.2.2" resolved "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" @@ -25139,6 +25210,25 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" +testcontainers@^7.10.0: + version "7.10.0" + resolved "https://registry.npmjs.org/testcontainers/-/testcontainers-7.10.0.tgz#d0306ac9e54c24006e0de0e4505ca7d4a5a0db9e" + integrity sha512-xwz1dzRUmaYTNL4qH8//YlJzSGfVmooGtJNc0ynFm0mtUTyYu4HJLpPZB9e7+/ZiQlWp73LSTH/gSwuNtl8FXw== + dependencies: + "@types/archiver" "^5.1.0" + "@types/dockerode" "^3.2.1" + archiver "^5.2.0" + byline "^5.0.0" + debug "^4.3.1" + docker-compose "^0.23.5" + dockerode "^3.2.1" + get-port "^5.1.1" + glob "^7.1.6" + slash "^3.0.0" + ssh-remote-port-forward "^1.0.3" + stream-to-array "^2.3.0" + tar-fs "^2.1.1" + text-encoding-utf-8@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz#585b62197b0ae437e3c7b5d0af27ac1021e10d13" @@ -27093,7 +27183,7 @@ yaml-ast-parser@0.0.43, yaml-ast-parser@^0.0.43: resolved "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb" integrity sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A== -yaml@^1.10.0, yaml@^1.7.2, yaml@^1.9.2: +yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2, yaml@^1.9.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== @@ -27275,6 +27365,15 @@ zip-stream@^4.0.4: compress-commons "^4.0.2" readable-stream "^3.6.0" +zip-stream@^4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79" + integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A== + dependencies: + archiver-utils "^2.1.0" + compress-commons "^4.1.0" + readable-stream "^3.6.0" + zombie@^6.1.4: version "6.1.4" resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47"