From 8165184fbaf0f96aade1624e4dfa25e5ead37a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 15 May 2026 17:15:23 +0200 Subject: [PATCH] tests: use describe.each for database test iteration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactors all test files that use TestDatabases/TestCaches with it.each(databases.eachSupportedId()) to instead use describe.each at the outer level. This ensures that all tests for one database engine complete before moving to the next, rather than interleaving engines across individual tests. This reduces the number of concurrent database connections and should help with test timeout issues in CI. The TestDatabases.create() call is hoisted to module scope so the describe.each can iterate over supported IDs at the top level. 40 files changed across packages/backend-defaults, packages/backend-test-utils, and multiple plugins. Signed-off-by: Fredrik Adelöw Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .../plugin/keys/createPluginKeySource.test.ts | 109 +- .../entrypoints/cache/CacheManager.test.ts | 82 +- .../scheduler/database/migrations.test.ts | 105 +- .../lib/DefaultSchedulerService.test.ts | 103 +- .../lib/PluginTaskSchedulerImpl.test.ts | 466 ++-- .../lib/PluginTaskSchedulerJanitor.test.ts | 45 +- .../backend-defaults/src/migrations.test.ts | 120 +- .../src/cache/TestCaches.test.ts | 6 +- .../src/database/TestDatabases.test.ts | 26 +- .../src/lib/assets/StaticAssetsStore.test.ts | 37 +- plugins/app-backend/src/migrations.test.ts | 144 +- .../src/identity/DatabaseKeyStore.test.ts | 23 +- .../src/identity/KeyStores.test.ts | 141 +- plugins/auth-backend/src/migrations.test.ts | 680 +++--- ...ncrementalIngestionDatabaseManager.test.ts | 35 +- .../src/module/WrapperProviders.test.ts | 39 +- .../database/DefaultCatalogDatabase.test.ts | 82 +- .../DefaultProcessingDatabase.test.ts | 262 +- .../database/DefaultProviderDatabase.test.ts | 221 +- .../src/database/metrics.test.ts | 177 +- .../deleteWithEagerPruningOfChildren.test.ts | 173 +- .../provider/refreshByRefreshKeys.test.ts | 30 +- .../getDeferredStitchableEntities.test.ts | 16 +- .../markDeferredStitchCompleted.test.ts | 16 +- .../stitcher/markForStitching.test.ts | 30 +- .../stitcher/performStitching.test.ts | 35 +- .../util/deleteOrphanedEntities.test.ts | 210 +- .../providers/DefaultLocationStore.test.ts | 294 +-- .../service/DefaultEntitiesCatalog.test.ts | 601 ++--- .../src/service/DefaultRefreshService.test.ts | 319 ++- .../src/service/createRouter.test.ts | 106 +- .../src/stitching/DefaultStitcher.test.ts | 454 ++-- .../src/tests/migrations.test.ts | 2160 ++++++++--------- .../performance/stitchingPerformance.test.ts | 24 +- plugins/events-backend/src/migrations.test.ts | 80 +- .../src/service/EventsPlugin.test.ts | 494 ++-- .../service/hub/DatabaseEventBusStore.test.ts | 45 +- .../src/tests/migrations.test.ts | 87 +- .../database/DatabaseDocumentStore.test.ts | 1247 +++++----- .../src/database/util.test.ts | 74 +- 40 files changed, 4406 insertions(+), 4992 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts b/packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts index 68ac322da5..8d7eae04c4 100644 --- a/packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts +++ b/packages/backend-defaults/src/entrypoints/auth/plugin/keys/createPluginKeySource.test.ts @@ -24,13 +24,13 @@ import { ConfigReader } from '@backstage/config'; jest.setTimeout(60_000); -describe('createPluginKeySource', () => { - const databases = TestDatabases.create(); - const mockDir = createMockDirectory(); +const databases = TestDatabases.create(); +const mockDir = createMockDirectory(); - it.each(databases.eachSupportedId())( - 'works for implicit database (no config), %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'createPluginKeySource, %p', + databaseId => { + it('works for implicit database (no config)', async () => { const knex = await databases.init(databaseId); const getClient = jest.fn(async () => knex); @@ -61,12 +61,9 @@ describe('createPluginKeySource', () => { }), ], }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'works for explicit database, %p', - async databaseId => { + it('works for explicit database', async () => { const knex = await databases.init(databaseId); const getClient = jest.fn(async () => knex); @@ -99,11 +96,12 @@ describe('createPluginKeySource', () => { }), ], }); - }, - ); + }); + }, +); - it('works for static', async () => { - const privateKey = ` +it('works for static', async () => { + const privateKey = ` -----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgR8Ja2ppMEgOm1KeY Kpje00U1luybndt6yC263vcgeKqhRANCAAS+slUrS9JXgtHB1RcDnmlveuu4H3Zm @@ -111,60 +109,59 @@ describe('createPluginKeySource', () => { -----END PRIVATE KEY----- `.trim(); - const publicKey = ` + const publicKey = ` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvrJVK0vSV4LRwdUXA55pb3rruB92 ZoUEY72HTvjIP9xSbLOhWhMREk84T0q91/m3v3sWq5EzHIWw1zRHQisKZw== -----END PUBLIC KEY----- `.trim(); - mockDir.setContent({ - 'public.pem': publicKey, - 'private.pem': privateKey, - }); - const publicKeyPath = mockDir.resolve('public.pem'); - const privateKeyPath = mockDir.resolve('private.pem'); + mockDir.setContent({ + 'public.pem': publicKey, + 'private.pem': privateKey, + }); + const publicKeyPath = mockDir.resolve('public.pem'); + const privateKeyPath = mockDir.resolve('private.pem'); - const getClient = jest.fn(); + const getClient = jest.fn(); - const source = await createPluginKeySource({ - config: new ConfigReader({ - backend: { - auth: { - pluginKeyStore: { - type: 'static', - static: { - keys: [ - { - publicKeyFile: publicKeyPath, - privateKeyFile: privateKeyPath, - keyId: '1', - }, - ], - }, + const source = await createPluginKeySource({ + config: new ConfigReader({ + backend: { + auth: { + pluginKeyStore: { + type: 'static', + static: { + keys: [ + { + publicKeyFile: publicKeyPath, + privateKeyFile: privateKeyPath, + keyId: '1', + }, + ], }, }, }, - }), - database: mockServices.database.mock({ getClient }), - logger: mockServices.logger.mock(), - keyDuration: { seconds: 10 }, - }); + }, + }), + database: mockServices.database.mock({ getClient }), + logger: mockServices.logger.mock(), + keyDuration: { seconds: 10 }, + }); - expect(getClient).not.toHaveBeenCalled(); + expect(getClient).not.toHaveBeenCalled(); - const keys = await source.listKeys(); - expect(keys.keys.length).toEqual(1); - expect(keys.keys[0].key).toMatchObject({ - kid: '1', - alg: 'ES256', - }); + const keys = await source.listKeys(); + expect(keys.keys.length).toEqual(1); + expect(keys.keys[0].key).toMatchObject({ + kid: '1', + alg: 'ES256', + }); - const pk = await source.getPrivateSigningKey(); - expect(pk).toMatchObject({ - kid: '1', - alg: 'ES256', - d: expect.any(String), - }); + const pk = await source.getPrivateSigningKey(); + expect(pk).toMatchObject({ + kid: '1', + alg: 'ES256', + d: expect.any(String), }); }); diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts index 323ca881c5..84844f45cc 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts @@ -54,14 +54,16 @@ jest.mock('@keyv/memcache', () => { }; }); -describe('CacheManager integration', () => { - const caches = TestCaches.create(); +jest.setTimeout(60_000); - afterEach(jest.clearAllMocks); +const caches = TestCaches.create(); - it.each(caches.eachSupportedId())( - 'only creates one underlying connection per plugin, %p', - async cacheId => { +describe.each(caches.eachSupportedId())( + 'CacheManager integration, %p', + cacheId => { + afterEach(jest.clearAllMocks); + + it('only creates one underlying connection per plugin', async () => { const { store, connection } = await caches.init(cacheId); const manager = CacheManager.fromConfig( @@ -85,12 +87,9 @@ describe('CacheManager integration', () => { // eslint-disable-next-line jest/no-conditional-expect expect(KeyvValkey).toHaveBeenCalledTimes(3); } - }, - ); + }); - it.each(caches.eachSupportedId())( - 'interacts correctly with store, %p', - async cacheId => { + it('interacts correctly with store', async () => { const { store, connection } = await caches.init(cacheId); const manager = CacheManager.fromConfig( @@ -114,12 +113,9 @@ describe('CacheManager integration', () => { await expect(plugin1.get('a')).resolves.toBe('plugin1'); await expect(plugin2a.get('a')).resolves.toBe('plugin2b'); await expect(plugin2b.get('a')).resolves.toBe('plugin2b'); - }, - ); + }); - it.each(caches.eachSupportedId())( - 'supports both milliseconds and human durations throughout, %p', - async cacheId => { + it('supports both milliseconds and human durations throughout', async () => { const { store, connection } = await caches.init(cacheId); for (const defaultTtl of [200, { milliseconds: 200 }]) { @@ -175,39 +171,39 @@ describe('CacheManager integration', () => { await expect(defaultClient.get('e')).resolves.toBeUndefined(); await expect(defaultClient.get('f')).resolves.toBeUndefined(); } - }, - ); + }); + }, +); - it('rejects invalid defaultTtl', () => { - expect(() => - CacheManager.fromConfig( - mockServices.rootConfig({ - data: { - backend: { - cache: { - store: 'memory', - }, +it('rejects invalid defaultTtl', () => { + expect(() => + CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'memory', }, }, - }), - ), - ).not.toThrow(); + }, + }), + ), + ).not.toThrow(); - expect(() => - CacheManager.fromConfig( - mockServices.rootConfig({ - data: { - backend: { - cache: { - store: 'memory', - defaultTtl: 'hello', - }, + expect(() => + CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'memory', + defaultTtl: 'hello', }, }, - }), - ), - ).toThrow(/Invalid duration 'hello' in config/); - }); + }, + }), + ), + ).toThrow(/Invalid duration 'hello' in config/); }); describe('CacheManager store options', () => { diff --git a/packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts b/packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts index 3764dacfb0..62337cc59f 100644 --- a/packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts +++ b/packages/backend-defaults/src/entrypoints/scheduler/database/migrations.test.ts @@ -41,68 +41,65 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - '20250411000000_last_run.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20250411000000_last_run.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20250411000000_last_run.js'); + await migrateUntilBefore(knex, '20250411000000_last_run.js'); - await knex - .insert({ - id: 'i', - settings_json: '{}', - }) - .into('backstage_backend_tasks__tasks'); + await knex + .insert({ + id: 'i', + settings_json: '{}', + }) + .into('backstage_backend_tasks__tasks'); - await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ - { - id: 'i', - settings_json: '{}', - next_run_start_at: null, - current_run_ticket: null, - current_run_started_at: null, - current_run_expires_at: null, - }, - ]); + await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ + { + id: 'i', + settings_json: '{}', + next_run_start_at: null, + current_run_ticket: null, + current_run_started_at: null, + current_run_expires_at: null, + }, + ]); - await migrateUpOnce(knex); + await migrateUpOnce(knex); - await knex - .table('backstage_backend_tasks__tasks') - .update({ last_run_error_json: 'error' }) - .where({ id: 'i' }); + await knex + .table('backstage_backend_tasks__tasks') + .update({ last_run_error_json: 'error' }) + .where({ id: 'i' }); - await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ - { - id: 'i', - settings_json: '{}', - next_run_start_at: null, - current_run_ticket: null, - current_run_started_at: null, - current_run_expires_at: null, - last_run_ended_at: null, - last_run_error_json: 'error', - }, - ]); + await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ + { + id: 'i', + settings_json: '{}', + next_run_start_at: null, + current_run_ticket: null, + current_run_started_at: null, + current_run_expires_at: null, + last_run_ended_at: null, + last_run_error_json: 'error', + }, + ]); - await migrateDownOnce(knex); + await migrateDownOnce(knex); - await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ - { - id: 'i', - settings_json: '{}', - next_run_start_at: null, - current_run_ticket: null, - current_run_started_at: null, - current_run_expires_at: null, - }, - ]); + await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ + { + id: 'i', + settings_json: '{}', + next_run_start_at: null, + current_run_ticket: null, + current_run_started_at: null, + current_run_expires_at: null, + }, + ]); - await knex.destroy(); - }, - ); + await knex.destroy(); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts b/packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts index 3de69ba566..d6d391c67f 100644 --- a/packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts +++ b/packages/backend-defaults/src/entrypoints/scheduler/lib/DefaultSchedulerService.test.ts @@ -24,9 +24,10 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha'; jest.setTimeout(60_000); -describe('TaskScheduler', () => { +const databases = TestDatabases.create(); + +describe.each(databases.eachSupportedId())('TaskScheduler, %p', databaseId => { const logger = mockServices.logger.mock(); - const databases = TestDatabases.create(); const rootLifecycle = mockServices.rootLifecycle.mock(); const httpRouter = mockServices.httpRouter.mock(); const pluginMetadata = { @@ -35,63 +36,57 @@ describe('TaskScheduler', () => { const testScopedSignal = createTestScopedSignal(); const metrics = metricsServiceMock.mock(); - it.each(databases.eachSupportedId())( - 'can return a working v1 plugin impl, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const database = mockServices.database({ knex }); + it('can return a working v1 plugin impl', async () => { + const knex = await databases.init(databaseId); + const database = mockServices.database({ knex }); - const manager = DefaultSchedulerService.create({ - database, - logger, - metrics, - rootLifecycle, - httpRouter, - pluginMetadata, - }); - const fn = jest.fn(); + const manager = DefaultSchedulerService.create({ + database, + logger, + metrics, + rootLifecycle, + httpRouter, + pluginMetadata, + }); + const fn = jest.fn(); - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromMillis(5000), - signal: testScopedSignal(), - fn, - }); + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromMillis(5000), + signal: testScopedSignal(), + fn, + }); - await waitForExpect(() => { - expect(fn).toHaveBeenCalled(); - }); - }, - ); + await waitForExpect(() => { + expect(fn).toHaveBeenCalled(); + }); + }); - it.each(databases.eachSupportedId())( - 'can return a working v2 plugin impl, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const database = mockServices.database({ knex }); + it('can return a working v2 plugin impl', async () => { + const knex = await databases.init(databaseId); + const database = mockServices.database({ knex }); - const manager = DefaultSchedulerService.create({ - database, - logger, - metrics, - rootLifecycle, - httpRouter, - pluginMetadata, - }); - const fn = jest.fn(); + const manager = DefaultSchedulerService.create({ + database, + logger, + metrics, + rootLifecycle, + httpRouter, + pluginMetadata, + }); + const fn = jest.fn(); - await manager.scheduleTask({ - id: 'task2', - timeout: Duration.fromMillis(5000), - frequency: { cron: '* * * * * *' }, - signal: testScopedSignal(), - fn, - }); + await manager.scheduleTask({ + id: 'task2', + timeout: Duration.fromMillis(5000), + frequency: { cron: '* * * * * *' }, + signal: testScopedSignal(), + fn, + }); - await waitForExpect(() => { - expect(fn).toHaveBeenCalled(); - }); - }, - ); + await waitForExpect(() => { + expect(fn).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts index c91466f667..60589e3237 100644 --- a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts +++ b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts @@ -31,48 +31,50 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha'; jest.setTimeout(60_000); -describe('PluginTaskManagerImpl', () => { - const addShutdownHook = jest.fn(); - const databases = TestDatabases.create({ - ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], - }); +const databases = TestDatabases.create({ + ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], +}); - beforeAll(async () => { - // Make sure all databases are running before mocking timers, in case of testcontainers - await Promise.all( - databases.eachSupportedId().map(([id]) => databases.init(id)), - ); +describe.each(databases.eachSupportedId())( + 'PluginTaskManagerImpl, %p', + databaseId => { + const addShutdownHook = jest.fn(); - jest.useFakeTimers(); - }, 60_000); + beforeAll(async () => { + // Make sure the database is running before mocking timers, in case of testcontainers + await databases.init(databaseId); + jest.useFakeTimers(); + }, 60_000); - beforeEach(() => { - jest.clearAllMocks(); - }); + afterAll(() => { + jest.useRealTimers(); + }); - async function init(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await migrateBackendTasks(knex); - const manager = new PluginTaskSchedulerImpl( - 'myplugin', - async () => knex, - mockServices.logger.mock(), - metricsServiceMock.mock(), - { - addShutdownHook, - addBeforeShutdownHook: jest.fn(), - addStartupHook: jest.fn(), - }, - ); - return { knex, manager }; - } + beforeEach(() => { + jest.clearAllMocks(); + }); - // This is just to test the wrapper code; most of the actual tests are in - // TaskWorker.test.ts - describe('scheduleTask with global scope', () => { - it.each(databases.eachSupportedId())( - 'can run the v1 happy path, %p', - async databaseId => { + async function init(id: TestDatabaseId) { + const knex = await databases.init(id); + await migrateBackendTasks(knex); + const manager = new PluginTaskSchedulerImpl( + 'myplugin', + async () => knex, + mockServices.logger.mock(), + metricsServiceMock.mock(), + { + addShutdownHook, + addBeforeShutdownHook: jest.fn(), + addStartupHook: jest.fn(), + }, + ); + return { knex, manager }; + } + + // This is just to test the wrapper code; most of the actual tests are in + // TaskWorker.test.ts + describe('scheduleTask with global scope', () => { + it('can run the v1 happy path', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -87,12 +89,9 @@ describe('PluginTaskManagerImpl', () => { await promise; expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can run the v2 happy path, %p', - async databaseId => { + it('can run the v2 happy path', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -107,12 +106,9 @@ describe('PluginTaskManagerImpl', () => { await promise; expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'aborts the task if shutdown hook is invoked, %p', - async databaseId => { + it('aborts the task if shutdown hook is invoked', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -134,14 +130,11 @@ describe('PluginTaskManagerImpl', () => { // Should be aborted after the shutdown hook is invoked await shutdownHook(); expect(abortSignal.aborted).toBe(true); - }, - ); - }); + }); + }); - describe('triggerTask with global scope', () => { - it.each(databases.eachSupportedId())( - 'can manually trigger a task, %p', - async databaseId => { + describe('triggerTask with global scope', () => { + it('can manually trigger a task', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -160,12 +153,9 @@ describe('PluginTaskManagerImpl', () => { await promise; expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'cant trigger a non-existent task, %p', - async databaseId => { + it('cant trigger a non-existent task', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -180,12 +170,9 @@ describe('PluginTaskManagerImpl', () => { await expect(() => manager.triggerTask('task2')).rejects.toThrow( NotFoundError, ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'cant trigger a running task, %p', - async databaseId => { + it('cant trigger a running task', async () => { const { manager } = await init(databaseId); const promise = createDeferred(); @@ -205,140 +192,137 @@ describe('PluginTaskManagerImpl', () => { await expect(() => manager.triggerTask('task1')).rejects.toThrow( ConflictError, ); - }, - ); - }); - - // This is just to test the wrapper code; most of the actual tests are in - // TaskWorker.test.ts - describe('scheduleTask with local scope', () => { - it('can run the v1 happy path', async () => { - const { manager } = await init('SQLITE_3'); - - const fn = jest.fn(); - const promise = new Promise(resolve => fn.mockImplementation(resolve)); - await manager.scheduleTask({ - id: 'task1', - timeout: { milliseconds: 5000 }, - frequency: { milliseconds: 5000 }, - fn, - scope: 'local', }); + }); - await promise; - expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, 60_000); + // This is just to test the wrapper code; most of the actual tests are in + // TaskWorker.test.ts + describe('scheduleTask with local scope', () => { + it('can run the v1 happy path', async () => { + const { manager } = await init('SQLITE_3'); - it('can run the v2 happy path', async () => { - const { manager } = await init('SQLITE_3'); + const fn = jest.fn(); + const promise = new Promise(resolve => fn.mockImplementation(resolve)); + await manager.scheduleTask({ + id: 'task1', + timeout: { milliseconds: 5000 }, + frequency: { milliseconds: 5000 }, + fn, + scope: 'local', + }); - const fn = jest.fn(); - const promise = new Promise(resolve => fn.mockImplementation(resolve)); - await manager.scheduleTask({ - id: 'task2', - timeout: Duration.fromMillis(5000), - frequency: { cron: '* * * * * *' }, - fn, - scope: 'local', - }); + await promise; + expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); + }, 60_000); - await promise; - expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, 60_000); + it('can run the v2 happy path', async () => { + const { manager } = await init('SQLITE_3'); - it('aborts the task if shutdown hook is invoked', async () => { - const { manager } = await init('SQLITE_3'); + const fn = jest.fn(); + const promise = new Promise(resolve => fn.mockImplementation(resolve)); + await manager.scheduleTask({ + id: 'task2', + timeout: Duration.fromMillis(5000), + frequency: { cron: '* * * * * *' }, + fn, + scope: 'local', + }); - const fn = jest.fn(); - const promise = new Promise(resolve => - fn.mockImplementation(resolve), - ); - await manager.scheduleTask({ - id: 'task3', - timeout: Duration.fromMillis(5000), - frequency: { cron: '* * * * * *' }, - fn, - scope: 'local', - }); + await promise; + expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); + }, 60_000); - const shutdownHook = addShutdownHook.mock.calls[0][0]; - const abortSignal = await promise; - expect(abortSignal.aborted).toBe(false); + it('aborts the task if shutdown hook is invoked', async () => { + const { manager } = await init('SQLITE_3'); - // Should be aborted after the shutdown hook is invoked - await shutdownHook(); - expect(abortSignal.aborted).toBe(true); - }, 60_000); - }); + const fn = jest.fn(); + const promise = new Promise(resolve => + fn.mockImplementation(resolve), + ); + await manager.scheduleTask({ + id: 'task3', + timeout: Duration.fromMillis(5000), + frequency: { cron: '* * * * * *' }, + fn, + scope: 'local', + }); - describe('triggerTask with local scope', () => { - it('can manually trigger a task', async () => { - const { manager } = await init('SQLITE_3'); + const shutdownHook = addShutdownHook.mock.calls[0][0]; + const abortSignal = await promise; + expect(abortSignal.aborted).toBe(false); - const fn = jest.fn(); - const promise = new Promise(resolve => fn.mockImplementation(resolve)); - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromObject({ years: 1 }), - initialDelay: Duration.fromObject({ years: 1 }), - fn, - scope: 'local', - }); + // Should be aborted after the shutdown hook is invoked + await shutdownHook(); + expect(abortSignal.aborted).toBe(true); + }, 60_000); + }); - await manager.triggerTask('task1'); - jest.advanceTimersByTime(5000); + describe('triggerTask with local scope', () => { + it('can manually trigger a task', async () => { + const { manager } = await init('SQLITE_3'); - await promise; - expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, 60_000); + const fn = jest.fn(); + const promise = new Promise(resolve => fn.mockImplementation(resolve)); + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromObject({ years: 1 }), + initialDelay: Duration.fromObject({ years: 1 }), + fn, + scope: 'local', + }); - it('cant trigger a non-existent task', async () => { - const { manager } = await init('SQLITE_3'); + await manager.triggerTask('task1'); + jest.advanceTimersByTime(5000); - const fn = jest.fn(); - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromObject({ years: 1 }), - fn, - scope: 'local', - }); + await promise; + expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); + }, 60_000); - await expect(() => manager.triggerTask('task2')).rejects.toThrow( - NotFoundError, - ); - }, 60_000); + it('cant trigger a non-existent task', async () => { + const { manager } = await init('SQLITE_3'); - it('cant trigger a running task', async () => { - const { manager } = await init('SQLITE_3'); + const fn = jest.fn(); + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromObject({ years: 1 }), + fn, + scope: 'local', + }); - const promise = createDeferred(); + await expect(() => manager.triggerTask('task2')).rejects.toThrow( + NotFoundError, + ); + }, 60_000); - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromObject({ years: 1 }), - fn: async () => { - promise.resolve(); - await new Promise(r => setTimeout(r, 20000)); - }, - scope: 'local', - }); + it('cant trigger a running task', async () => { + const { manager } = await init('SQLITE_3'); - await promise; - await expect(() => manager.triggerTask('task1')).rejects.toThrow( - ConflictError, - ); - }, 60_000); - }); + const promise = createDeferred(); - // This is just to test the wrapper code; most of the actual tests are in - // TaskWorker.test.ts - describe('createScheduledTaskRunner', () => { - it.each(databases.eachSupportedId())( - 'can run the happy path, %p', - async databaseId => { + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromObject({ years: 1 }), + fn: async () => { + promise.resolve(); + await new Promise(r => setTimeout(r, 20000)); + }, + scope: 'local', + }); + + await promise; + await expect(() => manager.triggerTask('task1')).rejects.toThrow( + ConflictError, + ); + }, 60_000); + }); + + // This is just to test the wrapper code; most of the actual tests are in + // TaskWorker.test.ts + describe('createScheduledTaskRunner', () => { + it('can run the happy path', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -356,14 +340,11 @@ describe('PluginTaskManagerImpl', () => { await promise; expect(fn).toHaveBeenCalledWith(expect.any(AbortSignal)); - }, - ); - }); + }); + }); - describe('can fetch task ids', () => { - it.each(databases.eachSupportedId())( - 'can fetch both global and local task ids, %p', - async databaseId => { + describe('can fetch task ids', () => { + it('can fetch both global and local task ids', async () => { const { manager } = await init(databaseId); const fn = jest.fn(); @@ -395,52 +376,51 @@ describe('PluginTaskManagerImpl', () => { settings: expect.objectContaining({ cadence: 'PT5S' }), }, ]); - }, - ); - }); - - describe('cancelTask with local scope', () => { - it('can cancel a running task', async () => { - const { manager } = await init('SQLITE_3'); - - const promise = createDeferred(); - - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromObject({ years: 1 }), - fn: async () => { - promise.resolve(); - await new Promise(r => setTimeout(r, 20000)); - }, - scope: 'local', }); + }); - await promise; - await expect(manager.cancelTask('task1')).resolves.toBeUndefined(); - }, 60_000); + describe('cancelTask with local scope', () => { + it('can cancel a running task', async () => { + const { manager } = await init('SQLITE_3'); - it('cannot cancel a task that is not running', async () => { - const { manager } = await init('SQLITE_3'); + const promise = createDeferred(); - const fn = jest.fn(); - await manager.scheduleTask({ - id: 'task1', - timeout: Duration.fromMillis(5000), - frequency: Duration.fromObject({ years: 1 }), - initialDelay: Duration.fromObject({ years: 1 }), - fn, - scope: 'local', - }); + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromObject({ years: 1 }), + fn: async () => { + promise.resolve(); + await new Promise(r => setTimeout(r, 20000)); + }, + scope: 'local', + }); - await expect(manager.cancelTask('task1')).rejects.toThrow(ConflictError); - }, 60_000); - }); + await promise; + await expect(manager.cancelTask('task1')).resolves.toBeUndefined(); + }, 60_000); - describe('cancelTask with global scope', () => { - it.each(databases.eachSupportedId())( - 'can cancel a running task, %p', - async databaseId => { + it('cannot cancel a task that is not running', async () => { + const { manager } = await init('SQLITE_3'); + + const fn = jest.fn(); + await manager.scheduleTask({ + id: 'task1', + timeout: Duration.fromMillis(5000), + frequency: Duration.fromObject({ years: 1 }), + initialDelay: Duration.fromObject({ years: 1 }), + fn, + scope: 'local', + }); + + await expect(manager.cancelTask('task1')).rejects.toThrow( + ConflictError, + ); + }, 60_000); + }); + + describe('cancelTask with global scope', () => { + it('can cancel a running task', async () => { const { manager } = await init(databaseId); const promise = createDeferred(); @@ -458,23 +438,17 @@ describe('PluginTaskManagerImpl', () => { await promise; await expect(manager.cancelTask('task1')).resolves.toBeUndefined(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'cannot cancel a non-existent task, %p', - async databaseId => { + it('cannot cancel a non-existent task', async () => { const { manager } = await init(databaseId); await expect(manager.cancelTask('nonexistent')).rejects.toThrow( NotFoundError, ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'cannot cancel a task that is not running, %p', - async databaseId => { + it('cannot cancel a task that is not running', async () => { const { manager } = await init(databaseId); await manager.scheduleTask({ @@ -489,16 +463,16 @@ describe('PluginTaskManagerImpl', () => { await expect(manager.cancelTask('task1')).rejects.toThrow( ConflictError, ); - }, - ); - }); - - describe('parseDuration', () => { - it('should parse durations', () => { - expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S'); - expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S'); - expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *'); - expect(parseDuration({ trigger: 'manual' })).toEqual('manual'); + }); }); - }); -}); + + describe('parseDuration', () => { + it('should parse durations', () => { + expect(parseDuration({ milliseconds: 5000 })).toEqual('PT5S'); + expect(parseDuration(Duration.fromMillis(5000))).toEqual('PT5S'); + expect(parseDuration({ cron: '1 * * * *' })).toEqual('1 * * * *'); + expect(parseDuration({ trigger: 'manual' })).toEqual('manual'); + }); + }); + }, +); diff --git a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts index 8c2b54f69f..560554300f 100644 --- a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts +++ b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerJanitor.test.ts @@ -34,28 +34,29 @@ const getTask = async (knex: Knex): Promise => { return (await knex(DB_TASKS_TABLE))[0]; }; -describe('PluginTaskSchedulerJanitor', () => { - const logger = mockServices.logger.mock(); - const databases = TestDatabases.create({ - ids: [ - /* 'MYSQL_8' not supported yet */ - 'POSTGRES_18', - 'POSTGRES_14', - 'SQLITE_3', - 'MYSQL_8', - ], - }); - const testScopedSignal = createTestScopedSignal(); +jest.setTimeout(60_000); - jest.setTimeout(60_000); +const databases = TestDatabases.create({ + ids: [ + /* 'MYSQL_8' not supported yet */ + 'POSTGRES_18', + 'POSTGRES_14', + 'SQLITE_3', + 'MYSQL_8', + ], +}); - beforeEach(() => { - jest.resetAllMocks(); - }); +describe.each(databases.eachSupportedId())( + 'PluginTaskSchedulerJanitor, %p', + databaseId => { + const logger = mockServices.logger.mock(); + const testScopedSignal = createTestScopedSignal(); - it.each(databases.eachSupportedId())( - 'Should update date if current_run_expires_at expires, %p', - async databaseId => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('Should update date if current_run_expires_at expires', async () => { const knex = await databases.init(databaseId); await migrateBackendTasks(knex); @@ -92,6 +93,6 @@ describe('PluginTaskSchedulerJanitor', () => { }), ); }); - }, - ); -}); + }); + }, +); diff --git a/packages/backend-defaults/src/migrations.test.ts b/packages/backend-defaults/src/migrations.test.ts index f2ad3b7508..314f3cfbab 100644 --- a/packages/backend-defaults/src/migrations.test.ts +++ b/packages/backend-defaults/src/migrations.test.ts @@ -41,83 +41,77 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - '20210928160613_init.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20210928160613_init.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20210928160613_init.js'); - await migrateUpOnce(knex); + await migrateUntilBefore(knex, '20210928160613_init.js'); + await migrateUpOnce(knex); - await knex('backstage_backend_tasks__tasks').insert({ + await knex('backstage_backend_tasks__tasks').insert({ + id: 'test', + settings_json: '{}', + next_run_start_at: knex.fn.now(), + }); + + await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ + { id: 'test', settings_json: '{}', - next_run_start_at: knex.fn.now(), - }); + next_run_start_at: expect.anything(), + current_run_ticket: null, + current_run_started_at: null, + current_run_expires_at: null, + }, + ]); - await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ - { - id: 'test', - settings_json: '{}', - next_run_start_at: expect.anything(), - current_run_ticket: null, - current_run_started_at: null, - current_run_expires_at: null, - }, - ]); + await migrateDownOnce(knex); - await migrateDownOnce(knex); + // This looks odd - you might expect a .toThrow at the end but that + // actually is flaky for some reason specifically on sqlite when + // performing multiple runs in sequence + await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual( + expect.anything(), + ); - // This looks odd - you might expect a .toThrow at the end but that - // actually is flaky for some reason specifically on sqlite when - // performing multiple runs in sequence - await expect(knex('backstage_backend_tasks__tasks')).rejects.toEqual( - expect.anything(), - ); + await knex.destroy(); + }); - await knex.destroy(); - }, - ); + it('20240712211735_nullable_next_run.js', async () => { + const knex = await databases.init(databaseId); - it.each(databases.eachSupportedId())( - '20240712211735_nullable_next_run.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js'); + await migrateUpOnce(knex); - await migrateUntilBefore(knex, '20240712211735_nullable_next_run.js'); - await migrateUpOnce(knex); + await knex('backstage_backend_tasks__tasks').insert({ + id: 'test', + settings_json: '{}', + next_run_start_at: knex.raw('null'), + }); - await knex('backstage_backend_tasks__tasks').insert({ + await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ + { + id: 'test', + settings_json: '{}', + next_run_start_at: null, + current_run_ticket: null, + current_run_started_at: null, + current_run_expires_at: null, + }, + ]); + + await migrateDownOnce(knex); + + await expect( + knex('backstage_backend_tasks__tasks').insert({ id: 'test', settings_json: '{}', next_run_start_at: knex.raw('null'), - }); + }), + ).rejects.toEqual(expect.anything()); - await expect(knex('backstage_backend_tasks__tasks')).resolves.toEqual([ - { - id: 'test', - settings_json: '{}', - next_run_start_at: null, - current_run_ticket: null, - current_run_started_at: null, - current_run_expires_at: null, - }, - ]); - - await migrateDownOnce(knex); - - await expect( - knex('backstage_backend_tasks__tasks').insert({ - id: 'test', - settings_json: '{}', - next_run_start_at: knex.raw('null'), - }), - ).rejects.toEqual(expect.anything()); - - await knex.destroy(); - }, - ); + await knex.destroy(); + }); }); diff --git a/packages/backend-test-utils/src/cache/TestCaches.test.ts b/packages/backend-test-utils/src/cache/TestCaches.test.ts index 7d5ef69704..482dd72347 100644 --- a/packages/backend-test-utils/src/cache/TestCaches.test.ts +++ b/packages/backend-test-utils/src/cache/TestCaches.test.ts @@ -21,10 +21,10 @@ const itIfDocker = isDockerDisabledForTests() ? it.skip : it; jest.setTimeout(60_000); -describe('TestCaches', () => { - const caches = TestCaches.create(); +const caches = TestCaches.create(); - it.each(caches.eachSupportedId())('fires up a cache, %p', async cacheId => { +describe.each(caches.eachSupportedId())('TestCaches, %p', cacheId => { + it('fires up a cache', async () => { const { keyv } = await caches.init(cacheId); await keyv.set('test', 'value'); await expect(keyv.get('test')).resolves.toBe('value'); diff --git a/packages/backend-test-utils/src/database/TestDatabases.test.ts b/packages/backend-test-utils/src/database/TestDatabases.test.ts index fea7189aec..ce0d6e600e 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.test.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.test.ts @@ -18,24 +18,14 @@ import { TestDatabases } from './TestDatabases'; jest.setTimeout(60_000); -describe('TestDatabases', () => { - describe('each create', () => { - const dbs = TestDatabases.create(); +const dbs = TestDatabases.create(); - it.each(dbs.eachSupportedId())( - 'creates distinct %p databases', - async databaseId => { - if (!dbs.supports(databaseId)) { - return; - } - const db1 = await dbs.init(databaseId); - const db2 = await dbs.init(databaseId); - await db1.schema.createTable('a', table => table.string('x').primary()); - await db2.schema.createTable('a', table => table.string('y').primary()); - await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([ - { a: 1 }, - ]); - }, - ); +describe.each(dbs.eachSupportedId())('TestDatabases, %p', databaseId => { + it('creates distinct databases', async () => { + const db1 = await dbs.init(databaseId); + const db2 = await dbs.init(databaseId); + await db1.schema.createTable('a', table => table.string('x').primary()); + await db2.schema.createTable('a', table => table.string('y').primary()); + await expect(db1.select({ a: db1.raw('1') })).resolves.toEqual([{ a: 1 }]); }); }); diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts index bd7a01e5e8..c796d0acb2 100644 --- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts @@ -19,12 +19,12 @@ import { StaticAssetsStore } from './StaticAssetsStore'; jest.setTimeout(60_000); -describe('StaticAssetsStore', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'should store and retrieve assets, %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'StaticAssetsStore, %p', + databaseId => { + it('should store and retrieve assets', async () => { const knex = await databases.init(databaseId); const store = await StaticAssetsStore.create({ @@ -61,12 +61,9 @@ describe('StaticAssetsStore', () => { await expect( store.getAsset('does-not-exist.txt'), ).resolves.toBeUndefined(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should update assets timestamps, but not contents, %p', - async databaseId => { + it('should update assets timestamps, but not contents', async () => { const knex = await databases.init(databaseId); const store = await StaticAssetsStore.create({ @@ -112,12 +109,9 @@ describe('StaticAssetsStore', () => { const sameBar = await store.getAsset('bar'); expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should trim old assets, %p', - async databaseId => { + it('should trim old assets', async () => { const knex = await databases.init(databaseId); const store = await StaticAssetsStore.create({ @@ -157,12 +151,9 @@ describe('StaticAssetsStore', () => { await expect(store.getAsset('new')).resolves.toBeDefined(); await expect(store.getAsset('old')).resolves.toBeUndefined(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should isolate assets in namespace, %p', - async databaseId => { + it('should isolate assets in namespace', async () => { const knex = await databases.init(databaseId); const store = await StaticAssetsStore.create({ @@ -197,6 +188,6 @@ describe('StaticAssetsStore', () => { await otherStore.trimAssets({ maxAgeSeconds: 0 }); await expect(otherStore.getAsset('bar')).resolves.not.toBeDefined(); - }, - ); -}); + }); + }, +); diff --git a/plugins/app-backend/src/migrations.test.ts b/plugins/app-backend/src/migrations.test.ts index eb327e3f6a..41a84cce4e 100644 --- a/plugins/app-backend/src/migrations.test.ts +++ b/plugins/app-backend/src/migrations.test.ts @@ -41,101 +41,95 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - '20211229105307_init.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20211229105307_init.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20211229105307_init.js'); - await migrateUpOnce(knex); + await migrateUntilBefore(knex, '20211229105307_init.js'); + await migrateUpOnce(knex); - await knex('static_assets_cache').insert({ + await knex('static_assets_cache').insert({ + path: 'main.js', + content: Buffer.from('some-script'), + last_modified_at: knex.fn.now(), + }); + + await expect(knex('static_assets_cache')).resolves.toEqual([ + { path: 'main.js', content: Buffer.from('some-script'), - last_modified_at: knex.fn.now(), - }); + last_modified_at: expect.anything(), + }, + ]); - await expect(knex('static_assets_cache')).resolves.toEqual([ - { - path: 'main.js', - content: Buffer.from('some-script'), - last_modified_at: expect.anything(), - }, - ]); + await migrateDownOnce(knex); - await migrateDownOnce(knex); + // This looks odd - you might expect a .toThrow at the end but that + // actually is flaky for some reason specifically on sqlite when + // performing multiple runs in sequence + await expect(knex('static_assets_cache')).rejects.toEqual( + expect.anything(), + ); - // This looks odd - you might expect a .toThrow at the end but that - // actually is flaky for some reason specifically on sqlite when - // performing multiple runs in sequence - await expect(knex('static_assets_cache')).rejects.toEqual( - expect.anything(), - ); + await knex.destroy(); + }); - await knex.destroy(); - }, - ); + it('20240113144027_assets-namespace.js', async () => { + const knex = await databases.init(databaseId); - it.each(databases.eachSupportedId())( - '20240113144027_assets-namespace.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUntilBefore(knex, '20240113144027_assets-namespace.js'); - await migrateUntilBefore(knex, '20240113144027_assets-namespace.js'); + await knex('static_assets_cache').insert({ + path: 'main.js', + content: Buffer.from('some-script'), + last_modified_at: knex.fn.now(), + }); - await knex('static_assets_cache').insert({ + await migrateUpOnce(knex); + + await expect(knex('static_assets_cache')).resolves.toEqual([ + { path: 'main.js', content: Buffer.from('some-script'), - last_modified_at: knex.fn.now(), - }); + namespace: 'default', + last_modified_at: expect.anything(), + }, + ]); - await migrateUpOnce(knex); + await knex('static_assets_cache').insert({ + path: 'main.js', + content: Buffer.from('other-script'), + namespace: 'other', + last_modified_at: knex.fn.now(), + }); - await expect(knex('static_assets_cache')).resolves.toEqual([ - { - path: 'main.js', - content: Buffer.from('some-script'), - namespace: 'default', - last_modified_at: expect.anything(), - }, - ]); - - await knex('static_assets_cache').insert({ + await expect(knex('static_assets_cache')).resolves.toEqual([ + { + path: 'main.js', + content: Buffer.from('some-script'), + namespace: 'default', + last_modified_at: expect.anything(), + }, + { path: 'main.js', content: Buffer.from('other-script'), namespace: 'other', - last_modified_at: knex.fn.now(), - }); + last_modified_at: expect.anything(), + }, + ]); - await expect(knex('static_assets_cache')).resolves.toEqual([ - { - path: 'main.js', - content: Buffer.from('some-script'), - namespace: 'default', - last_modified_at: expect.anything(), - }, - { - path: 'main.js', - content: Buffer.from('other-script'), - namespace: 'other', - last_modified_at: expect.anything(), - }, - ]); + await migrateDownOnce(knex); - await migrateDownOnce(knex); + await expect(knex('static_assets_cache')).resolves.toEqual([ + { + path: 'main.js', + content: Buffer.from('some-script'), + last_modified_at: expect.anything(), + }, + ]); - await expect(knex('static_assets_cache')).resolves.toEqual([ - { - path: 'main.js', - content: Buffer.from('some-script'), - last_modified_at: expect.anything(), - }, - ]); - - await knex.destroy(); - }, - ); + await knex.destroy(); + }); }); diff --git a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts index f86c4a400b..02f1d2d7bd 100644 --- a/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts +++ b/plugins/auth-backend/src/identity/DatabaseKeyStore.test.ts @@ -27,12 +27,12 @@ const keyBase = { jest.setTimeout(60_000); -describe('DatabaseKeyStore', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'should store a key, %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'DatabaseKeyStore, %p', + databaseId => { + it('should store a key', async () => { const knex = await databases.init(databaseId); await AuthDatabase.runMigrations(knex); @@ -53,12 +53,9 @@ describe('DatabaseKeyStore', () => { DateTime.fromJSDate(items[0].createdAt).diffNow('seconds').seconds, ), ).toBeLessThan(10); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should remove stored keys, %p', - async databaseId => { + it('should remove stored keys', async () => { const knex = await databases.init(databaseId); await AuthDatabase.runMigrations(knex); @@ -124,6 +121,6 @@ describe('DatabaseKeyStore', () => { await expect(store.listKeys()).resolves.toEqual({ items: [], }); - }, - ); -}); + }); + }, +); diff --git a/plugins/auth-backend/src/identity/KeyStores.test.ts b/plugins/auth-backend/src/identity/KeyStores.test.ts index 13818ff79c..1110abd834 100644 --- a/plugins/auth-backend/src/identity/KeyStores.test.ts +++ b/plugins/auth-backend/src/identity/KeyStores.test.ts @@ -24,89 +24,80 @@ import { mockServices, TestDatabases } from '@backstage/backend-test-utils'; jest.setTimeout(60_000); -describe('KeyStores', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - const defaultConfigOptions = { - auth: { - keyStore: { - provider: 'memory', - }, +const defaultConfigOptions = { + auth: { + keyStore: { + provider: 'memory', }, - }; - const defaultConfig = new ConfigReader(defaultConfigOptions); + }, +}; +const defaultConfig = new ConfigReader(defaultConfigOptions); - it.each(databases.eachSupportedId())( - 'reads auth section from config, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig'); - const keyStore = await KeyStores.fromConfig(defaultConfig, { - logger: mockServices.logger.mock(), - database: AuthDatabase.create(mockServices.database({ knex })), - }); +describe.each(databases.eachSupportedId())('KeyStores, %p', databaseId => { + it('reads auth section from config', async () => { + const knex = await databases.init(databaseId); + const configSpy = jest.spyOn(defaultConfig, 'getOptionalConfig'); + const keyStore = await KeyStores.fromConfig(defaultConfig, { + logger: mockServices.logger.mock(), + database: AuthDatabase.create(mockServices.database({ knex })), + }); - expect(keyStore).toBeInstanceOf(MemoryKeyStore); - expect(configSpy).toHaveBeenCalledWith('auth.keyStore'); - expect( - defaultConfig - .getOptionalConfig('auth.keyStore') - ?.getOptionalString('provider'), - ).toBe(defaultConfigOptions.auth.keyStore.provider); - }, - ); + expect(keyStore).toBeInstanceOf(MemoryKeyStore); + expect(configSpy).toHaveBeenCalledWith('auth.keyStore'); + expect( + defaultConfig + .getOptionalConfig('auth.keyStore') + ?.getOptionalString('provider'), + ).toBe(defaultConfigOptions.auth.keyStore.provider); + }); - it.each(databases.eachSupportedId())( - 'can handle without auth config, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const keyStore = await KeyStores.fromConfig(new ConfigReader({}), { - logger: mockServices.logger.mock(), - database: AuthDatabase.create(mockServices.database({ knex })), - }); - expect(keyStore).toBeInstanceOf(DatabaseKeyStore); - }, - ); + it('can handle without auth config', async () => { + const knex = await databases.init(databaseId); + const keyStore = await KeyStores.fromConfig(new ConfigReader({}), { + logger: mockServices.logger.mock(), + database: AuthDatabase.create(mockServices.database({ knex })), + }); + expect(keyStore).toBeInstanceOf(DatabaseKeyStore); + }); - it.each(databases.eachSupportedId())( - 'can handle additional provider config, %p', - async databaseId => { - const knex = await databases.init(databaseId); - jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue(); - const createSpy = jest.spyOn(FirestoreKeyStore, 'create'); + it('can handle additional provider config', async () => { + const knex = await databases.init(databaseId); + jest.spyOn(FirestoreKeyStore, 'verifyConnection').mockResolvedValue(); + const createSpy = jest.spyOn(FirestoreKeyStore, 'create'); - const configOptions = { - auth: { - keyStore: { - provider: 'firestore', - firestore: { - projectId: 'my-project', - keyFilename: 'cred.json', - path: 'my-path', - timeout: 100, - host: 'localhost', - port: 8088, - ssl: false, - }, + const configOptions = { + auth: { + keyStore: { + provider: 'firestore', + firestore: { + projectId: 'my-project', + keyFilename: 'cred.json', + path: 'my-path', + timeout: 100, + host: 'localhost', + port: 8088, + ssl: false, }, }, - }; - const config = new ConfigReader(configOptions); - const keyStore = await KeyStores.fromConfig(config, { - logger: mockServices.logger.mock(), - database: AuthDatabase.create(mockServices.database({ knex })), - }); + }, + }; + const config = new ConfigReader(configOptions); + const keyStore = await KeyStores.fromConfig(config, { + logger: mockServices.logger.mock(), + database: AuthDatabase.create(mockServices.database({ knex })), + }); - expect(keyStore).toBeInstanceOf(FirestoreKeyStore); - expect(createSpy).toHaveBeenCalledWith( - configOptions.auth.keyStore.firestore, - ); - expect( - config - .getOptionalConfig('auth.keyStore') - ?.getOptionalConfig('firestore') - ?.getOptionalString('projectId'), - ).toBe(configOptions.auth.keyStore.firestore.projectId); - }, - ); + expect(keyStore).toBeInstanceOf(FirestoreKeyStore); + expect(createSpy).toHaveBeenCalledWith( + configOptions.auth.keyStore.firestore, + ); + expect( + config + .getOptionalConfig('auth.keyStore') + ?.getOptionalConfig('firestore') + ?.getOptionalString('projectId'), + ).toBe(configOptions.auth.keyStore.firestore.projectId); + }); }); diff --git a/plugins/auth-backend/src/migrations.test.ts b/plugins/auth-backend/src/migrations.test.ts index 5dec51ed85..eeb9cdbba8 100644 --- a/plugins/auth-backend/src/migrations.test.ts +++ b/plugins/auth-backend/src/migrations.test.ts @@ -41,179 +41,154 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - '20230428155633_sessions.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20230428155633_sessions.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20230428155633_sessions.js'); + await migrateUntilBefore(knex, '20230428155633_sessions.js'); + await migrateUpOnce(knex); + + // Ensure that large cookies are supported + const data = `{"cookie":"${'a'.repeat(100_000)}"}`; + await knex + .insert({ sid: 'abc', expired: knex.fn.now(), sess: data }) + .into('sessions'); + await knex + .insert({ sid: 'def', expired: knex.fn.now(), sess: data }) + .into('sessions'); + + await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([ + { sid: 'abc', expired: expect.anything(), sess: data }, + { sid: 'def', expired: expect.anything(), sess: data }, + ]); + + await migrateDownOnce(knex); + + await knex.destroy(); + }); + + it('20240510120825_user_info.js', async () => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore(knex, '20240510120825_user_info.js'); + await migrateUpOnce(knex); + + const user_info = JSON.stringify({ + claims: { + ent: ['group:default/group1', 'group:default/group2'], + }, + }); + + await knex + .insert({ + user_entity_ref: 'user:default/backstage-user', + user_info, + exp: knex.fn.now(), + }) + .into('user_info'); + + await expect(knex('user_info')).resolves.toEqual([ + { + user_entity_ref: 'user:default/backstage-user', + user_info, + exp: expect.anything(), + }, + ]); + + await migrateDownOnce(knex); + + await knex.destroy(); + }); + + it('20250707164600_user_created_at.js', async () => { + const knex = await databases.init(databaseId); + await migrateUntilBefore(knex, '20250707164600_user_created_at.js'); + + if (knex.client.config.client.includes('sqlite')) { + // Sqlite doesn't support adding a column with non-constant default when table has data + // so we just test that the migration runs without errors await migrateUpOnce(knex); - // Ensure that large cookies are supported - const data = `{"cookie":"${'a'.repeat(100_000)}"}`; - await knex - .insert({ sid: 'abc', expired: knex.fn.now(), sess: data }) - .into('sessions'); - await knex - .insert({ sid: 'def', expired: knex.fn.now(), sess: data }) - .into('sessions'); + return; + } - await expect(knex('sessions').orderBy('sid', 'asc')).resolves.toEqual([ - { sid: 'abc', expired: expect.anything(), sess: data }, - { sid: 'def', expired: expect.anything(), sess: data }, - ]); + const user_info = JSON.stringify({ + claims: { + ent: ['group:default/group1', 'group:default/group2'], + }, + }); - await migrateDownOnce(knex); + await knex + .insert({ + user_entity_ref: 'user:default/backstage-user', + user_info, + exp: knex.fn.now(), + }) + .into('user_info'); - await knex.destroy(); - }, - ); + const { exp } = await knex('user_info').first(); - it.each(databases.eachSupportedId())( - '20240510120825_user_info.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUpOnce(knex); - await migrateUntilBefore(knex, '20240510120825_user_info.js'); - await migrateUpOnce(knex); + const { created_at, updated_at } = await knex('user_info').first(); - const user_info = JSON.stringify({ - claims: { - ent: ['group:default/group1', 'group:default/group2'], - }, - }); + expect(updated_at).toEqual(exp); + expect(created_at).toBeDefined(); - await knex - .insert({ - user_entity_ref: 'user:default/backstage-user', - user_info, - exp: knex.fn.now(), - }) - .into('user_info'); + await knex + .insert({ + user_entity_ref: 'user:default/backstage-user', + user_info, + updated_at: knex.fn.now(), + }) + .into('user_info') + .onConflict(['user_entity_ref']) + .merge(); - await expect(knex('user_info')).resolves.toEqual([ - { - user_entity_ref: 'user:default/backstage-user', - user_info, - exp: expect.anything(), - }, - ]); + await knex + .insert({ + user_entity_ref: 'user:default/backstage-user-2', + user_info, + updated_at: knex.fn.now(), + }) + .into('user_info'); - await migrateDownOnce(knex); + await expect( + knex('user_info').select('created_at', 'updated_at'), + ).resolves.toEqual([ + { + created_at: expect.any(Date), + updated_at: expect.any(Date), + }, + { + created_at: expect.any(Date), + updated_at: expect.any(Date), + }, + ]); - await knex.destroy(); - }, - ); + await migrateDownOnce(knex); - it.each(databases.eachSupportedId())( - '20250707164600_user_created_at.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20250707164600_user_created_at.js'); + await expect(knex('user_info').select('exp')).resolves.toEqual([ + { exp: expect.any(Date) }, + { exp: expect.any(Date) }, + ]); - if (knex.client.config.client.includes('sqlite')) { - // Sqlite doesn't support adding a column with non-constant default when table has data - // so we just test that the migration runs without errors - await migrateUpOnce(knex); + await knex.destroy(); + }); - return; - } + it('20250909120000_oidc_client_registration.js', async () => { + const knex = await databases.init(databaseId); - const user_info = JSON.stringify({ - claims: { - ent: ['group:default/group1', 'group:default/group2'], - }, - }); + await migrateUntilBefore( + knex, + '20250909120000_oidc_client_registration.js', + ); + await migrateUpOnce(knex); - await knex - .insert({ - user_entity_ref: 'user:default/backstage-user', - user_info, - exp: knex.fn.now(), - }) - .into('user_info'); - - const { exp } = await knex('user_info').first(); - - await migrateUpOnce(knex); - - const { created_at, updated_at } = await knex('user_info').first(); - - expect(updated_at).toEqual(exp); - expect(created_at).toBeDefined(); - - await knex - .insert({ - user_entity_ref: 'user:default/backstage-user', - user_info, - updated_at: knex.fn.now(), - }) - .into('user_info') - .onConflict(['user_entity_ref']) - .merge(); - - await knex - .insert({ - user_entity_ref: 'user:default/backstage-user-2', - user_info, - updated_at: knex.fn.now(), - }) - .into('user_info'); - - await expect( - knex('user_info').select('created_at', 'updated_at'), - ).resolves.toEqual([ - { - created_at: expect.any(Date), - updated_at: expect.any(Date), - }, - { - created_at: expect.any(Date), - updated_at: expect.any(Date), - }, - ]); - - await migrateDownOnce(knex); - - await expect(knex('user_info').select('exp')).resolves.toEqual([ - { exp: expect.any(Date) }, - { exp: expect.any(Date) }, - ]); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20250909120000_oidc_client_registration.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await migrateUntilBefore( - knex, - '20250909120000_oidc_client_registration.js', - ); - await migrateUpOnce(knex); - - await knex - .insert({ - client_id: 'test-client-id', - client_secret: 'test-client-secret', - client_name: 'Test Client', - response_types: JSON.stringify(['code']), - grant_types: JSON.stringify(['authorization_code']), - redirect_uris: JSON.stringify(['https://example.com/callback']), - scope: 'openid profile', - metadata: JSON.stringify({ description: 'Test client' }), - }) - .into('oidc_clients'); - - await expect( - knex('oidc_clients').where('client_id', 'test-client-id').first(), - ).resolves.toEqual({ + await knex + .insert({ client_id: 'test-client-id', client_secret: 'test-client-secret', client_name: 'Test Client', @@ -222,230 +197,235 @@ describe('migrations', () => { redirect_uris: JSON.stringify(['https://example.com/callback']), scope: 'openid profile', metadata: JSON.stringify({ description: 'Test client' }), - }); + }) + .into('oidc_clients'); - await knex - .insert({ - id: 'test-session-id', - client_id: 'test-client-id', - user_entity_ref: 'user:default/test-user', - redirect_uri: 'https://example.com/callback', - scope: 'openid', - state: 'test-state', - response_type: 'code', - code_challenge: 'test-challenge', - code_challenge_method: 'S256', - nonce: 'test-nonce', - status: 'pending', - expires_at: new Date(Date.now() + 3600000), - }) - .into('oauth_authorization_sessions'); + await expect( + knex('oidc_clients').where('client_id', 'test-client-id').first(), + ).resolves.toEqual({ + client_id: 'test-client-id', + client_secret: 'test-client-secret', + client_name: 'Test Client', + response_types: JSON.stringify(['code']), + grant_types: JSON.stringify(['authorization_code']), + redirect_uris: JSON.stringify(['https://example.com/callback']), + scope: 'openid profile', + metadata: JSON.stringify({ description: 'Test client' }), + }); - await expect( - knex('oauth_authorization_sessions') - .where('id', 'test-session-id') - .first(), - ).resolves.toEqual( - expect.objectContaining({ - id: 'test-session-id', - client_id: 'test-client-id', - user_entity_ref: 'user:default/test-user', - redirect_uri: 'https://example.com/callback', - scope: 'openid', - state: 'test-state', - response_type: 'code', - code_challenge: 'test-challenge', - code_challenge_method: 'S256', - nonce: 'test-nonce', - status: 'pending', - }), - ); + await knex + .insert({ + id: 'test-session-id', + client_id: 'test-client-id', + user_entity_ref: 'user:default/test-user', + redirect_uri: 'https://example.com/callback', + scope: 'openid', + state: 'test-state', + response_type: 'code', + code_challenge: 'test-challenge', + code_challenge_method: 'S256', + nonce: 'test-nonce', + status: 'pending', + expires_at: new Date(Date.now() + 3600000), + }) + .into('oauth_authorization_sessions'); - await knex - .insert({ - code: 'test-auth-code', - session_id: 'test-session-id', - expires_at: new Date(Date.now() + 600000), - used: false, - }) - .into('oidc_authorization_codes'); - - await expect( - knex('oidc_authorization_codes') - .where('code', 'test-auth-code') - .first(), - ).resolves.toEqual( - expect.objectContaining({ - code: 'test-auth-code', - session_id: 'test-session-id', - }), - ); - - await knex('oauth_authorization_sessions') + await expect( + knex('oauth_authorization_sessions') .where('id', 'test-session-id') - .del(); + .first(), + ).resolves.toEqual( + expect.objectContaining({ + id: 'test-session-id', + client_id: 'test-client-id', + user_entity_ref: 'user:default/test-user', + redirect_uri: 'https://example.com/callback', + scope: 'openid', + state: 'test-state', + response_type: 'code', + code_challenge: 'test-challenge', + code_challenge_method: 'S256', + nonce: 'test-nonce', + status: 'pending', + }), + ); - await expect( - knex('oidc_authorization_codes').where('session_id', 'test-session-id'), - ).resolves.toHaveLength(0); + await knex + .insert({ + code: 'test-auth-code', + session_id: 'test-session-id', + expires_at: new Date(Date.now() + 600000), + used: false, + }) + .into('oidc_authorization_codes'); - await migrateDownOnce(knex); + await expect( + knex('oidc_authorization_codes').where('code', 'test-auth-code').first(), + ).resolves.toEqual( + expect.objectContaining({ + code: 'test-auth-code', + session_id: 'test-session-id', + }), + ); - const tables = [ - 'oidc_clients', - 'oauth_authorization_sessions', - 'oidc_authorization_codes', - ]; + await knex('oauth_authorization_sessions') + .where('id', 'test-session-id') + .del(); - for (const table of tables) { - await expect(knex.schema.hasTable(table)).resolves.toBe(false); - } + await expect( + knex('oidc_authorization_codes').where('session_id', 'test-session-id'), + ).resolves.toHaveLength(0); - await knex.destroy(); - }, - ); + await migrateDownOnce(knex); - it.each(databases.eachSupportedId())( - '20251118120000_oauth_state_text.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + const tables = [ + 'oidc_clients', + 'oauth_authorization_sessions', + 'oidc_authorization_codes', + ]; - await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js'); + for (const table of tables) { + await expect(knex.schema.hasTable(table)).resolves.toBe(false); + } - // First create a client for the foreign key constraint - await knex - .insert({ - client_id: 'test-client-id', - client_secret: 'test-client-secret', - client_name: 'Test Client', - response_types: JSON.stringify(['code']), - grant_types: JSON.stringify(['authorization_code']), - redirect_uris: JSON.stringify(['https://example.com/callback']), - }) - .into('oidc_clients'); + await knex.destroy(); + }); - // Insert a session with state before migration - const existingState = 'existing-short-state'; - await knex - .insert({ - id: 'test-existing-session', - client_id: 'test-client-id', - redirect_uri: 'https://example.com/callback', - state: existingState, - response_type: 'code', - status: 'pending', - expires_at: new Date(Date.now() + 3600000), - }) - .into('oauth_authorization_sessions'); + it('20251118120000_oauth_state_text.js', async () => { + const knex = await databases.init(databaseId); - // Apply the migration that changes state to TEXT - await migrateUpOnce(knex); + await migrateUntilBefore(knex, '20251118120000_oauth_state_text.js'); - // Verify existing state persists after migration - await expect( - knex('oauth_authorization_sessions') - .where('id', 'test-existing-session') - .first(), - ).resolves.toEqual( - expect.objectContaining({ - id: 'test-existing-session', - state: existingState, - }), - ); + // First create a client for the foreign key constraint + await knex + .insert({ + client_id: 'test-client-id', + client_secret: 'test-client-secret', + client_name: 'Test Client', + response_types: JSON.stringify(['code']), + grant_types: JSON.stringify(['authorization_code']), + redirect_uris: JSON.stringify(['https://example.com/callback']), + }) + .into('oidc_clients'); - // Test inserting a state parameter longer than 255 characters - // This is based on the real-world example from the issue - const longState = 'a'.repeat(280); + // Insert a session with state before migration + const existingState = 'existing-short-state'; + await knex + .insert({ + id: 'test-existing-session', + client_id: 'test-client-id', + redirect_uri: 'https://example.com/callback', + state: existingState, + response_type: 'code', + status: 'pending', + expires_at: new Date(Date.now() + 3600000), + }) + .into('oauth_authorization_sessions'); - await knex - .insert({ - id: 'test-long-state-session', - client_id: 'test-client-id', - redirect_uri: 'https://example.com/callback', - state: longState, - response_type: 'code', - status: 'pending', - expires_at: new Date(Date.now() + 3600000), - }) - .into('oauth_authorization_sessions'); + // Apply the migration that changes state to TEXT + await migrateUpOnce(knex); - await expect( - knex('oauth_authorization_sessions') - .where('id', 'test-long-state-session') - .first(), - ).resolves.toEqual( - expect.objectContaining({ - id: 'test-long-state-session', - state: longState, - }), - ); + // Verify existing state persists after migration + await expect( + knex('oauth_authorization_sessions') + .where('id', 'test-existing-session') + .first(), + ).resolves.toEqual( + expect.objectContaining({ + id: 'test-existing-session', + state: existingState, + }), + ); - await migrateDownOnce(knex); + // Test inserting a state parameter longer than 255 characters + // This is based on the real-world example from the issue + const longState = 'a'.repeat(280); - await knex.destroy(); - }, - ); + await knex + .insert({ + id: 'test-long-state-session', + client_id: 'test-client-id', + redirect_uri: 'https://example.com/callback', + state: longState, + response_type: 'code', + status: 'pending', + expires_at: new Date(Date.now() + 3600000), + }) + .into('oauth_authorization_sessions'); - it.each(databases.eachSupportedId())( - '20251217120000_drop_oidc_clients_fk.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await expect( + knex('oauth_authorization_sessions') + .where('id', 'test-long-state-session') + .first(), + ).resolves.toEqual( + expect.objectContaining({ + id: 'test-long-state-session', + state: longState, + }), + ); - await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js'); + await migrateDownOnce(knex); - // Create a client for DCR sessions - await knex - .insert({ - client_id: 'dcr-client-id', - client_secret: 'test-client-secret', - client_name: 'DCR Client', - response_types: JSON.stringify(['code']), - grant_types: JSON.stringify(['authorization_code']), - redirect_uris: JSON.stringify(['https://example.com/callback']), - }) - .into('oidc_clients'); + await knex.destroy(); + }); - // Create a DCR session (has matching client in oidc_clients) - await knex - .insert({ - id: 'dcr-session', - client_id: 'dcr-client-id', - redirect_uri: 'https://example.com/callback', - response_type: 'code', - status: 'pending', - expires_at: new Date(Date.now() + 3600000), - }) - .into('oauth_authorization_sessions'); + it('20251217120000_drop_oidc_clients_fk.js', async () => { + const knex = await databases.init(databaseId); - // Apply migration - drops FK constraint - await migrateUpOnce(knex); + await migrateUntilBefore(knex, '20251217120000_drop_oidc_clients_fk.js'); - // Now we can insert a CIMD session (URL-based client_id not in oidc_clients) - await knex - .insert({ - id: 'cimd-session', - client_id: 'https://example.com/.well-known/oauth-client/cli', - redirect_uri: 'http://localhost:8080/callback', - response_type: 'code', - status: 'pending', - expires_at: new Date(Date.now() + 3600000), - }) - .into('oauth_authorization_sessions'); + // Create a client for DCR sessions + await knex + .insert({ + client_id: 'dcr-client-id', + client_secret: 'test-client-secret', + client_name: 'DCR Client', + response_types: JSON.stringify(['code']), + grant_types: JSON.stringify(['authorization_code']), + redirect_uris: JSON.stringify(['https://example.com/callback']), + }) + .into('oidc_clients'); - // Verify both sessions exist - await expect( - knex('oauth_authorization_sessions').select('id').orderBy('id'), - ).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]); + // Create a DCR session (has matching client in oidc_clients) + await knex + .insert({ + id: 'dcr-session', + client_id: 'dcr-client-id', + redirect_uri: 'https://example.com/callback', + response_type: 'code', + status: 'pending', + expires_at: new Date(Date.now() + 3600000), + }) + .into('oauth_authorization_sessions'); - // Rollback - should delete CIMD sessions and re-add FK - await migrateDownOnce(knex); + // Apply migration - drops FK constraint + await migrateUpOnce(knex); - // CIMD session should be deleted, DCR session should remain - await expect( - knex('oauth_authorization_sessions').select('id'), - ).resolves.toEqual([{ id: 'dcr-session' }]); + // Now we can insert a CIMD session (URL-based client_id not in oidc_clients) + await knex + .insert({ + id: 'cimd-session', + client_id: 'https://example.com/.well-known/oauth-client/cli', + redirect_uri: 'http://localhost:8080/callback', + response_type: 'code', + status: 'pending', + expires_at: new Date(Date.now() + 3600000), + }) + .into('oauth_authorization_sessions'); - await knex.destroy(); - }, - ); + // Verify both sessions exist + await expect( + knex('oauth_authorization_sessions').select('id').orderBy('id'), + ).resolves.toEqual([{ id: 'cimd-session' }, { id: 'dcr-session' }]); + + // Rollback - should delete CIMD sessions and re-add FK + await migrateDownOnce(knex); + + // CIMD session should be deleted, DCR session should remain + await expect( + knex('oauth_authorization_sessions').select('id'), + ).resolves.toEqual([{ id: 'dcr-session' }]); + + await knex.destroy(); + }); }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts index a99ca1cb94..035fc8d419 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.test.ts @@ -23,18 +23,20 @@ const migrationsDir = `${__dirname}/../../migrations`; jest.setTimeout(60_000); -describe('IncrementalIngestionDatabaseManager', () => { - const databases = TestDatabases.create({ - ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], - }); +const databases = TestDatabases.create({ + ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], +}); - it.each(databases.eachSupportedId())( - 'stores and retrieves marks, %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'IncrementalIngestionDatabaseManager, %p', + databaseId => { + it('stores and retrieves marks', async () => { const knex = await databases.init(databaseId); await knex.migrate.latest({ directory: migrationsDir }); - const manager = new IncrementalIngestionDatabaseManager({ client: knex }); + const manager = new IncrementalIngestionDatabaseManager({ + client: knex, + }); const { ingestionId } = (await manager.createProviderIngestionRecord( 'myProvider', ))!; @@ -75,16 +77,15 @@ describe('IncrementalIngestionDatabaseManager', () => { sequence: 1, }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'computeRemoved correctly sums total count from count query, %p', - async databaseId => { + it('computeRemoved correctly sums total count from count query', async () => { const knex = await databases.init(databaseId); await knex.migrate.latest({ directory: migrationsDir }); - const manager = new IncrementalIngestionDatabaseManager({ client: knex }); + const manager = new IncrementalIngestionDatabaseManager({ + client: knex, + }); const { ingestionId } = (await manager.createProviderIngestionRecord( 'testProvider', ))!; @@ -119,6 +120,6 @@ describe('IncrementalIngestionDatabaseManager', () => { // On PostgreSQL, count queries return strings, so total should be 3 not NaN or string concatenation expect(result.total).toBe(3); expect(typeof result.total).toBe('number'); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index f4b40ffff4..dfb443f0dc 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -23,24 +23,25 @@ import { WrapperProviders } from './WrapperProviders'; jest.setTimeout(60_000); -describe('WrapperProviders', () => { - const applyDatabaseMigrations = jest.fn(); - const databases = TestDatabases.create({ - ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'], - }); - const config = new ConfigReader({}); - const logger = mockServices.logger.mock(); - const scheduler = { - scheduleTask: jest.fn(), - }; +const databases = TestDatabases.create({ + ids: ['POSTGRES_18', 'POSTGRES_14', 'SQLITE_3', 'MYSQL_8'], +}); - beforeEach(() => { - jest.clearAllMocks(); - }); +describe.each(databases.eachSupportedId())( + 'WrapperProviders, %p', + databaseId => { + const applyDatabaseMigrations = jest.fn(); + const config = new ConfigReader({}); + const logger = mockServices.logger.mock(); + const scheduler = { + scheduleTask: jest.fn(), + }; - it.each(databases.eachSupportedId())( - 'should initialize the providers in order, %p', - async databaseId => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should initialize the providers in order', async () => { const client = await databases.init(databaseId); const provider1: IncrementalEntityProvider = { @@ -111,6 +112,6 @@ describe('WrapperProviders', () => { id: 'provider2', }), ); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts index a537168bcf..2f25336775 100644 --- a/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - mockServices, - TestDatabaseId, - TestDatabases, -} from '@backstage/backend-test-utils'; +import { mockServices, TestDatabases } from '@backstage/backend-test-utils'; import { DefaultCatalogDatabase } from './DefaultCatalogDatabase'; import { applyDatabaseMigrations } from './migrations'; import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; @@ -26,48 +22,46 @@ import { LoggerService } from '@backstage/backend-plugin-api'; jest.setTimeout(60_000); -describe('DefaultCatalogDatabase', () => { - const defaultLogger = mockServices.logger.mock(); - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase( - databaseId: TestDatabaseId, - logger: LoggerService = defaultLogger, - ) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return { - knex, - db: new DefaultCatalogDatabase({ - database: knex, - logger, - }), - }; - } +describe.each(databases.eachSupportedId())( + 'DefaultCatalogDatabase, %p', + databaseId => { + const defaultLogger = mockServices.logger.mock(); - describe('listAncestors', () => { - let nextId = 1; - function makeEntity(ref: string) { + async function createDatabase(logger: LoggerService = defaultLogger) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); return { - entity_id: String(nextId++), - entity_ref: ref, - unprocessed_entity: JSON.stringify({ - kind: 'Location', - apiVersion: '1.0.0', - metadata: { - name: 'xyz', - }, + knex, + db: new DefaultCatalogDatabase({ + database: knex, + logger, }), - errors: '[]', - next_update_at: '2019-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', }; } - it.each(databases.eachSupportedId())( - 'should return ancestors, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + describe('listAncestors', () => { + let nextId = 1; + function makeEntity(ref: string) { + return { + entity_id: String(nextId++), + entity_ref: ref, + unprocessed_entity: JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + }), + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }; + } + + it('should return ancestors', async () => { + const { knex, db } = await createDatabase(); await knex('refresh_state').insert( makeEntity('location:default/root-1'), @@ -107,7 +101,7 @@ describe('DefaultCatalogDatabase', () => { 'location:default/root-1', 'location:default/root-2', ]); - }, - ); - }); -}); + }); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index d1501c073d..4b6811f183 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - mockServices, - TestDatabaseId, - TestDatabases, -} from '@backstage/backend-test-utils'; +import { mockServices, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { randomUUID as uuid } from 'node:crypto'; @@ -40,64 +36,62 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha'; jest.setTimeout(60_000); -describe('DefaultProcessingDatabase', () => { - const defaultLogger = mockServices.logger.mock(); - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase( - databaseId: TestDatabaseId, - logger: LoggerService = defaultLogger, - ) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return { - knex, - db: new DefaultProcessingDatabase({ - database: knex, - logger, - refreshInterval: createRandomProcessingInterval({ - minSeconds: 100, - maxSeconds: 150, +describe.each(databases.eachSupportedId())( + 'DefaultProcessingDatabase, %p', + databaseId => { + const defaultLogger = mockServices.logger.mock(); + + async function createDatabase(logger: LoggerService = defaultLogger) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return { + knex, + db: new DefaultProcessingDatabase({ + database: knex, + logger, + refreshInterval: createRandomProcessingInterval({ + minSeconds: 100, + maxSeconds: 150, + }), + events: mockServices.events.mock(), + metrics: metricsServiceMock.mock(), }), - events: mockServices.events.mock(), - metrics: metricsServiceMock.mock(), - }), - }; - } - - const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { - return db('refresh_state_references').insert( - ref, - ); - }; - - const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { - await db('refresh_state').insert(ref); - }; - - describe('updateProcessedEntity', () => { - let id: string; - let processedEntity: Entity; - - beforeEach(() => { - id = uuid(); - processedEntity = { - apiVersion: '1', - kind: 'Location', - metadata: { - name: 'fakelocation', - }, - spec: { - type: 'url', - target: 'somethingelse', - }, }; - }); + } - it.each(databases.eachSupportedId())( - 'fails when an entity is processed with a different locationKey, %p', - async databaseId => { - const { db } = await createDatabase(databaseId); + const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { + return db('refresh_state_references').insert( + ref, + ); + }; + + const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { + await db('refresh_state').insert(ref); + }; + + describe('updateProcessedEntity', () => { + let id: string; + let processedEntity: Entity; + + beforeEach(() => { + id = uuid(); + processedEntity = { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'fakelocation', + }, + spec: { + type: 'url', + target: 'somethingelse', + }, + }; + }); + + it('fails when an entity is processed with a different locationKey', async () => { + const { db } = await createDatabase(); await db.transaction(async tx => { await expect(() => db.updateProcessedEntity(tx, { @@ -112,12 +106,9 @@ describe('DefaultProcessingDatabase', () => { `Conflicting write of processing result for ${id} with location key 'undefined'`, ); }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'fails when the locationKey is different, %p', - async databaseId => { + it('fails when the locationKey is different', async () => { const options = { id, processedEntity, @@ -128,7 +119,7 @@ describe('DefaultProcessingDatabase', () => { refreshKeys: [], errors: "['something broke']", }; - const { knex, db } = await createDatabase(databaseId); + const { knex, db } = await createDatabase(); await insertRefreshStateRow(knex, { entity_id: id, entity_ref: 'location:default/fakelocation', @@ -158,13 +149,10 @@ describe('DefaultProcessingDatabase', () => { `Conflicting write of processing result for ${id} with location key 'fail'`, ), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'updates the refresh state entry with the cache, processed entity and errors, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('updates the refresh state entry with the cache, processed entity and errors', async () => { + const { knex, db } = await createDatabase(); await insertRefreshStateRow(knex, { entity_id: id, entity_ref: 'location:default/fakelocation', @@ -197,13 +185,10 @@ describe('DefaultProcessingDatabase', () => { ); expect(entities[0].errors).toEqual("['something broke']"); expect(entities[0].location_key).toEqual('key'); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'removes old relations and stores the new relationships, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('removes old relations and stores the new relationships', async () => { + const { knex, db } = await createDatabase(); await insertRefreshStateRow(knex, { entity_id: id, entity_ref: 'location:default/fakelocation', @@ -282,13 +267,10 @@ describe('DefaultProcessingDatabase', () => { target_entity_ref: 'component:default/foo', }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'adds deferred entities to the refresh_state table to be picked up later, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('adds deferred entities to the refresh_state table to be picked up later', async () => { + const { knex, db } = await createDatabase(); await insertRefreshStateRow(knex, { entity_id: id, entity_ref: 'location:default/fakelocation', @@ -330,19 +312,15 @@ describe('DefaultProcessingDatabase', () => { .select(); expect(refreshStateEntries).toHaveLength(1); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'updates unprocessed entities with varying location keys, %p', - async databaseId => { + it('updates unprocessed entities with varying location keys', async () => { const mockLogger = { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), }; const { knex, db } = await createDatabase( - databaseId, mockLogger as unknown as Logger, ); @@ -479,19 +457,15 @@ describe('DefaultProcessingDatabase', () => { expect(mockLogger.error).not.toHaveBeenCalled(); } }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'stores the refresh keys for the entity where key length is 255 chars or less', - async databaseId => { + it('stores the refresh keys for the entity where key length is 255 chars or less', async () => { const mockLogger = { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), }; const { knex, db } = await createDatabase( - databaseId, mockLogger as unknown as Logger, ); await insertRefreshStateRow(knex, { @@ -536,19 +510,15 @@ describe('DefaultProcessingDatabase', () => { entity_id: id, key: 'protocol:foo-bar.com', }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'stores the refresh keys for the entity where key length is greater than 255 chars', - async databaseId => { + it('stores the refresh keys for the entity where key length is greater than 255 chars', async () => { const mockLogger = { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), }; const { knex, db } = await createDatabase( - databaseId, mockLogger as unknown as Logger, ); await insertRefreshStateRow(knex, { @@ -597,15 +567,12 @@ describe('DefaultProcessingDatabase', () => { entity_id: id, key: `url:https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-back#sha256:edfb606500d184900e63891e5279d35bf0069ea251e90d15c0a430de6023d905`, }); - }, - ); - }); + }); + }); - describe('updateEntityCache', () => { - it.each(databases.eachSupportedId())( - 'updates the entityCache, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + describe('updateEntityCache', () => { + it('updates the entityCache', async () => { + const { knex, db } = await createDatabase(); const id = '123'; await insertRefreshStateRow(knex, { entity_id: id, @@ -644,15 +611,12 @@ describe('DefaultProcessingDatabase', () => { ).select(); expect(entities2.length).toBe(1); expect(entities2[0].cache).toEqual('{}'); - }, - ); - }); + }); + }); - describe('getProcessableEntities', () => { - it.each(databases.eachSupportedId())( - 'should return entities to process, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + describe('getProcessableEntities', () => { + it('should return entities to process', async () => { + const { knex, db } = await createDatabase(); const entity = JSON.stringify({ kind: 'Location', apiVersion: '1.0.0', @@ -696,13 +660,10 @@ describe('DefaultProcessingDatabase', () => { }), ).resolves.toEqual({ items: [] }); }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should update the next_refresh interval with a timestamp that includes refresh spread, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should update the next_refresh interval with a timestamp that includes refresh spread', async () => { + const { knex, db } = await createDatabase(); const entity = JSON.stringify({ kind: 'Location', apiVersion: '1.0.0', @@ -731,33 +692,30 @@ describe('DefaultProcessingDatabase', () => { const nextUpdate = timestampToDateTime(result[0].next_update_at); const nextUpdateDiff = nextUpdate.diff(now, 'seconds'); expect(nextUpdateDiff.seconds).toBeGreaterThanOrEqual(90); - }, - ); - }); + }); + }); - describe('listParents', () => { - let nextId = 1; - function makeEntity(ref: string) { - return { - entity_id: String(nextId++), - entity_ref: ref, - unprocessed_entity: JSON.stringify({ - kind: 'Location', - apiVersion: '1.0.0', - metadata: { - name: 'xyz', - }, - }), - errors: '[]', - next_update_at: '2019-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', - }; - } + describe('listParents', () => { + let nextId = 1; + function makeEntity(ref: string) { + return { + entity_id: String(nextId++), + entity_ref: ref, + unprocessed_entity: JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + }), + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }; + } - it.each(databases.eachSupportedId())( - 'should return parents, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should return parents', async () => { + const { knex, db } = await createDatabase(); await knex('refresh_state').insert( makeEntity('location:default/root-1'), @@ -803,7 +761,7 @@ describe('DefaultProcessingDatabase', () => { db.listParents(tx, { entityRefs: ['location:default/root-2'] }), ); expect(result3.entityRefs).toEqual([]); - }, - ); - }); -}); + }); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index b193564102..2e2f3657cc 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - mockServices, - TestDatabaseId, - TestDatabases, -} from '@backstage/backend-test-utils'; +import { mockServices, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { randomUUID as uuid } from 'node:crypto'; @@ -30,54 +26,52 @@ import { generateStableHash } from './util'; jest.setTimeout(60_000); -describe('DefaultProviderDatabase', () => { - const defaultLogger = mockServices.logger.mock(); - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase( - databaseId: TestDatabaseId, - logger: LoggerService = defaultLogger, - ) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return { - knex, - db: new DefaultProviderDatabase({ - database: knex, - logger, - }), - }; - } +describe.each(databases.eachSupportedId())( + 'DefaultProviderDatabase, %p', + databaseId => { + const defaultLogger = mockServices.logger.mock(); - const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { - return db('refresh_state_references').insert( - ref, - ); - }; - - const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { - await db('refresh_state').insert(ref); - }; - - const createLocations = async (db: Knex, entityRefs: string[]) => { - for (const ref of entityRefs) { - await insertRefreshStateRow(db, { - entity_id: uuid(), - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); + async function createDatabase(logger: LoggerService = defaultLogger) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return { + knex, + db: new DefaultProviderDatabase({ + database: knex, + logger, + }), + }; } - }; - describe('replaceUnprocessedEntities', () => { - it.each(databases.eachSupportedId())( - 'replaces all existing state correctly for simple dependency chains, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { + return db('refresh_state_references').insert( + ref, + ); + }; + + const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { + await db('refresh_state').insert(ref); + }; + + const createLocations = async (db: Knex, entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow(db, { + entity_id: uuid(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + }; + + describe('replaceUnprocessedEntities', () => { + it('replaces all existing state correctly for simple dependency chains', async () => { + const { knex, db } = await createDatabase(); /* config -> location:default/root -> location:default/root-1 -> location:default/root-2 database -> location:default/second -> location:default/root-2 @@ -187,13 +181,10 @@ describe('DefaultProviderDatabase', () => { t.source_key === 'config', ), ).toBeTruthy(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should work for more complex chains, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should work for more complex chains', async () => { + const { knex, db } = await createDatabase(); /* config -> location:default/root -> location:default/root-1 -> location:default/root-2 config -> location:default/root -> location:default/root-1a -> location:default/root-2 @@ -323,13 +314,10 @@ describe('DefaultProviderDatabase', () => { t.target_entity_ref === 'location:default/root-2', ), ).toBeFalsy(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should add new locations using the delta options, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should add new locations using the delta options', async () => { + const { knex, db } = await createDatabase(); // Existing state and references should stay await createLocations(knex, ['location:default/existing']); @@ -393,13 +381,10 @@ describe('DefaultProviderDatabase', () => { t.target_entity_ref === 'location:default/existing', ), ).toBeTruthy(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should not remove locations that are referenced elsewhere, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should not remove locations that are referenced elsewhere', async () => { + const { knex, db } = await createDatabase(); /* config-1 -> location:default/root config-2 -> location:default/root @@ -443,13 +428,10 @@ describe('DefaultProviderDatabase', () => { entity_ref: 'location:default/root', }), ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should remove old locations using the delta options, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should remove old locations using the delta options', async () => { + const { knex, db } = await createDatabase(); await createLocations(knex, ['location:default/new-root']); await insertRefRow(knex, { @@ -492,13 +474,10 @@ describe('DefaultProviderDatabase', () => { t.target_entity_ref === 'location:default/new-root', ), ).toBeFalsy(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should update the location key during full replace, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should update the location key during full replace', async () => { + const { knex, db } = await createDatabase(); await createLocations(knex, ['location:default/removed']); await insertRefreshStateRow(knex, { entity_id: uuid(), @@ -558,13 +537,10 @@ describe('DefaultProviderDatabase', () => { target_entity_ref: 'location:default/replaced', }), ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should support replacing modified entities during a full update, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('should support replacing modified entities during a full update', async () => { + const { knex, db } = await createDatabase(); await db.transaction(async tx => { await db.replaceUnprocessedEntities(tx, { @@ -689,14 +665,11 @@ describe('DefaultProviderDatabase', () => { target_entity_ref: 'component:default/a', }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should successfully fall back from batch to individual mode on conflicts, %p', - async databaseId => { + it('should successfully fall back from batch to individual mode on conflicts', async () => { const fakeLogger = mockServices.logger.mock(); - const { knex, db } = await createDatabase(databaseId, fakeLogger); + const { knex, db } = await createDatabase(fakeLogger); await createLocations(knex, ['component:default/a']); @@ -738,14 +711,11 @@ describe('DefaultProviderDatabase', () => { }), ]), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync, %p', - async databaseId => { + it('should gracefully handle accidental duplicate refresh state references when deletion happens during a full sync', async () => { const fakeLogger = mockServices.logger.mock(); - const { knex, db } = await createDatabase(databaseId, fakeLogger); + const { knex, db } = await createDatabase(fakeLogger); await createLocations(knex, ['component:default/a']); @@ -768,14 +738,11 @@ describe('DefaultProviderDatabase', () => { const state = await knex('refresh_state').select(); expect(state).toEqual([]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should properly translate deltas into add/update/remove, %p', - async databaseId => { + it('should properly translate deltas into add/update/remove', async () => { const fakeLogger = mockServices.logger.mock(); - const { knex, db } = await createDatabase(databaseId, fakeLogger); + const { knex, db } = await createDatabase(fakeLogger); const entity1Before: Entity = { apiVersion: '1', @@ -950,14 +917,11 @@ describe('DefaultProviderDatabase', () => { location_key: 'new', // managed to update only the location key }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can handle large deltas without exploding, %p', - async databaseId => { + it('can handle large deltas without exploding', async () => { const fakeLogger = mockServices.logger.mock(); - const { knex, db } = await createDatabase(databaseId, fakeLogger); + const { knex, db } = await createDatabase(fakeLogger); const count = 10000; const padded = (n: number) => String(n).padStart(8, '0'); @@ -989,15 +953,12 @@ describe('DefaultProviderDatabase', () => { unprocessed_entity: JSON.stringify(entities[0].entity), unprocessed_hash: generateStableHash(entities[0].entity), }); - }, - ); - }); + }); + }); - describe('listReferenceSourceKeys', () => { - it.each(databases.eachSupportedId())( - 'returns the source_keys from "refresh_state_references", %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + describe('listReferenceSourceKeys', () => { + it('returns the source_keys from "refresh_state_references"', async () => { + const { knex, db } = await createDatabase(); await createLocations(knex, [ 'location:default/root', @@ -1018,13 +979,10 @@ describe('DefaultProviderDatabase', () => { ); expect(res).toEqual(['bar', 'foo']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'returns only unique source_keys", %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('returns only unique source_keys"', async () => { + const { knex, db } = await createDatabase(); await createLocations(knex, [ 'location:default/root', @@ -1045,13 +1003,10 @@ describe('DefaultProviderDatabase', () => { ); expect(res).toEqual(['foo']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'does not return null source_keys", %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); + it('does not return null source_keys"', async () => { + const { knex, db } = await createDatabase(); await createLocations(knex, [ 'location:default/root', @@ -1071,7 +1026,7 @@ describe('DefaultProviderDatabase', () => { ); expect(res).toEqual(['foo']); - }, - ); - }); -}); + }); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/metrics.test.ts b/plugins/catalog-backend/src/database/metrics.test.ts index 74613dcc91..a4f6a9bd6f 100644 --- a/plugins/catalog-backend/src/database/metrics.test.ts +++ b/plugins/catalog-backend/src/database/metrics.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { Knex } from 'knex'; import { randomUUID as uuid } from 'node:crypto'; import { applyDatabaseMigrations } from './migrations'; @@ -23,10 +23,10 @@ import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics'; jest.setTimeout(60_000); -describe('metrics', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase(databaseId: TestDatabaseId) { +describe.each(databases.eachSupportedId())('metrics, %p', databaseId => { + async function createDatabase() { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); return knex; @@ -55,108 +55,99 @@ describe('metrics', () => { } describe('queryEntitiesCountByKind', () => { - it.each(databases.eachSupportedId())( - 'counts entities grouped by the kind in entity_ref, %p', - async databaseId => { - const knex = await createDatabase(databaseId); + it('counts entities grouped by the kind in entity_ref', async () => { + const knex = await createDatabase(); - await insertEntity(knex, { - entityRef: 'component:default/svc-a', - finalEntity: '{"kind":"Component"}', - }); - await insertEntity(knex, { - entityRef: 'component:default/svc-b', - finalEntity: '{"kind":"Component"}', - }); - await insertEntity(knex, { - entityRef: 'api:default/api-a', - finalEntity: '{"kind":"API"}', - }); - await insertEntity(knex, { - entityRef: 'system:other/sys-a', - finalEntity: '{"kind":"System"}', - }); - // Not yet stitched -- must be excluded from the count - await insertEntity(knex, { - entityRef: 'component:default/pending', - finalEntity: null, - }); + await insertEntity(knex, { + entityRef: 'component:default/svc-a', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'component:default/svc-b', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'api:default/api-a', + finalEntity: '{"kind":"API"}', + }); + await insertEntity(knex, { + entityRef: 'system:other/sys-a', + finalEntity: '{"kind":"System"}', + }); + // Not yet stitched -- must be excluded from the count + await insertEntity(knex, { + entityRef: 'component:default/pending', + finalEntity: null, + }); - const result = await queryEntitiesCountByKind(knex); + const result = await queryEntitiesCountByKind(knex); - expect(Object.fromEntries(result)).toEqual({ - component: 2, - api: 1, - system: 1, - }); - }, - ); + expect(Object.fromEntries(result)).toEqual({ + component: 2, + api: 1, + system: 1, + }); + }); }); describe('createEntitiesCountByKind', () => { - it.each(databases.eachSupportedId())( - 'serves cached results within the TTL and refreshes after, %p', - async databaseId => { - const knex = await createDatabase(databaseId); - const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + it('serves cached results within the TTL and refreshes after', async () => { + const knex = await createDatabase(); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); - await insertEntity(knex, { - entityRef: 'component:default/one', - finalEntity: '{}', - }); + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); - const first = await getCount(); - expect(Object.fromEntries(first)).toEqual({ component: 1 }); + const first = await getCount(); + expect(Object.fromEntries(first)).toEqual({ component: 1 }); - // A change made within the TTL window must not be visible yet. - await insertEntity(knex, { - entityRef: 'component:default/two', - finalEntity: '{}', - }); - const cached = await getCount(); - expect(Object.fromEntries(cached)).toEqual({ component: 1 }); + // A change made within the TTL window must not be visible yet. + await insertEntity(knex, { + entityRef: 'component:default/two', + finalEntity: '{}', + }); + const cached = await getCount(); + expect(Object.fromEntries(cached)).toEqual({ component: 1 }); - // After the TTL elapses the next call hits the database again. - await new Promise(resolve => setTimeout(resolve, 80)); - const refreshed = await getCount(); - expect(Object.fromEntries(refreshed)).toEqual({ component: 2 }); - }, - ); + // After the TTL elapses the next call hits the database again. + await new Promise(resolve => setTimeout(resolve, 80)); + const refreshed = await getCount(); + expect(Object.fromEntries(refreshed)).toEqual({ component: 2 }); + }); - it.each(databases.eachSupportedId())( - 'coalesces overlapping callers into a single underlying query, %p', - async databaseId => { - const knex = await createDatabase(databaseId); - const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + it('coalesces overlapping callers into a single underlying query', async () => { + const knex = await createDatabase(); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); - await insertEntity(knex, { - entityRef: 'component:default/one', - finalEntity: '{}', - }); + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); - const finalEntitiesQueries: string[] = []; - knex.on('query', (q: { sql: string }) => { - if ( - /from\s+["`]?final_entities["`]?/i.test(q.sql) && - /^\s*select/i.test(q.sql) - ) { - finalEntitiesQueries.push(q.sql); - } - }); - - // Five concurrent callers should result in one query, not five. - const results = await Promise.all([ - getCount(), - getCount(), - getCount(), - getCount(), - getCount(), - ]); - expect(finalEntitiesQueries).toHaveLength(1); - for (const r of results) { - expect(Object.fromEntries(r)).toEqual({ component: 1 }); + const finalEntitiesQueries: string[] = []; + knex.on('query', (q: { sql: string }) => { + if ( + /from\s+["`]?final_entities["`]?/i.test(q.sql) && + /^\s*select/i.test(q.sql) + ) { + finalEntitiesQueries.push(q.sql); } - }, - ); + }); + + // Five concurrent callers should result in one query, not five. + const results = await Promise.all([ + getCount(), + getCount(), + getCount(), + getCount(), + getCount(), + ]); + expect(finalEntitiesQueries).toHaveLength(1); + for (const r of results) { + expect(Object.fromEntries(r)).toEqual({ component: 1 }); + } + }); }); }); diff --git a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts index 1ec0611347..b9a458a3ab 100644 --- a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts +++ b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { Knex } from 'knex'; import { randomUUID as uuid } from 'node:crypto'; import { applyDatabaseMigrations } from '../../migrations'; @@ -27,72 +27,72 @@ import { deleteWithEagerPruningOfChildren } from './deleteWithEagerPruningOfChil jest.setTimeout(60_000); -describe('deleteWithEagerPruningOfChildren', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return knex; - } - - async function insertReference( - knex: Knex, - ...refs: DbRefreshStateReferencesRow[] - ) { - return knex('refresh_state_references').insert( - refs, - ); - } - - async function insertRelation( - knex: Knex, - ...relations: { from: string; to: string }[] - ) { - for (const rel of relations) { - await knex('relations').insert({ - originating_entity_id: await knex('refresh_state') - .select('entity_id') - .then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion - source_entity_ref: rel.from, - target_entity_ref: rel.to, - type: 'fake', - }); +describe.each(databases.eachSupportedId())( + 'deleteWithEagerPruningOfChildren, %p', + databaseId => { + async function createDatabase() { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return knex; } - } - async function insertEntity(knex: Knex, ...entityRefs: string[]) { - for (const ref of entityRefs) { - await knex('refresh_state').insert({ - entity_id: uuid(), - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); + async function insertReference( + knex: Knex, + ...refs: DbRefreshStateReferencesRow[] + ) { + return knex( + 'refresh_state_references', + ).insert(refs); } - } - async function remainingEntities(knex: Knex) { - const rows = await knex('refresh_state') - .orderBy('entity_ref') - .select('entity_ref'); - return rows.map(r => r.entity_ref); - } + async function insertRelation( + knex: Knex, + ...relations: { from: string; to: string }[] + ) { + for (const rel of relations) { + await knex('relations').insert({ + originating_entity_id: await knex('refresh_state') + .select('entity_id') + .then(rows => rows[0].entity_id), // doesn't matter which one, this is consumed pre-deletion + source_entity_ref: rel.from, + target_entity_ref: rel.to, + type: 'fake', + }); + } + } - async function entitiesMarkedForStitching(knex: Knex) { - const rows = await knex('refresh_state') - .orderBy('entity_ref') - .select('entity_ref') - .where('result_hash', '=', 'force-stitching'); - return rows.map(r => r.entity_ref); - } + async function insertEntity(knex: Knex, ...entityRefs: string[]) { + for (const ref of entityRefs) { + await knex('refresh_state').insert({ + entity_id: uuid(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + } - it.each(databases.eachSupportedId())( - 'works for the simple path, %p', - async databaseId => { + async function remainingEntities(knex: Knex) { + const rows = await knex('refresh_state') + .orderBy('entity_ref') + .select('entity_ref'); + return rows.map(r => r.entity_ref); + } + + async function entitiesMarkedForStitching(knex: Knex) { + const rows = await knex('refresh_state') + .orderBy('entity_ref') + .select('entity_ref') + .where('result_hash', '=', 'force-stitching'); + return rows.map(r => r.entity_ref); + } + + it('works for the simple path', async () => { /* P1 - E1 - E2 @@ -106,7 +106,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: E1, E2, and E3 deleted; E4 and E5 remain; E4 marked for stitching because it had a relation to a deleted entity */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5'); await insertReference( knex, @@ -124,12 +124,9 @@ describe('deleteWithEagerPruningOfChildren', () => { }); await expect(remainingEntities(knex)).resolves.toEqual(['E4', 'E5']); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E4']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'works when there are multiple identical references, %p', - async databaseId => { + it('works when there are multiple identical references', async () => { /* P1 \ @@ -143,7 +140,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: E1 deleted; E2 remains; E2 marked for stitching because it had a relation to a deleted entity */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2'); await insertReference( knex, @@ -159,12 +156,9 @@ describe('deleteWithEagerPruningOfChildren', () => { }); await expect(remainingEntities(knex)).resolves.toEqual(['E2']); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'leaves out things that have roots in other source keys, %p', - async databaseId => { + it('leaves out things that have roots in other source keys', async () => { /* P1 - E1 \ @@ -176,7 +170,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: E1 deleted; E2 and E3 remain; E2 marked for stitching because it had a relation to a deleted entity */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2', 'E3'); await insertReference( knex, @@ -197,12 +191,9 @@ describe('deleteWithEagerPruningOfChildren', () => { }); await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual(['E2']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'leaves out things that have several different roots for the same source key, %p', - async databaseId => { + it('leaves out things that have several different roots for the same source key', async () => { /* P1 - E1 \ @@ -214,7 +205,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: E1 deleted; E2 and E3 remain */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2', 'E3'); await insertReference( knex, @@ -230,12 +221,9 @@ describe('deleteWithEagerPruningOfChildren', () => { }); await expect(remainingEntities(knex)).resolves.toEqual(['E2', 'E3']); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'handles cycles and diamonds gracefully, %p', - async databaseId => { + it('handles cycles and diamonds gracefully', async () => { /* P1 - E1 <-> E2 \ @@ -247,7 +235,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: Everything deleted, but in two steps; E4 marked for stitching in the first step because it had a relation to a deleted entity */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2', 'E3', 'E4', 'E5', 'E6'); await insertReference( knex, @@ -281,12 +269,9 @@ describe('deleteWithEagerPruningOfChildren', () => { }); await expect(remainingEntities(knex)).resolves.toEqual([]); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'silently ignores attempts to delete things that are not your own and/or are not roots, %p', - async databaseId => { + it('silently ignores attempts to delete things that are not your own and/or are not roots', async () => { /* P1 - E1 - E2 @@ -298,7 +283,7 @@ describe('deleteWithEagerPruningOfChildren', () => { Result: E3 is deleted; E1, E2 and E4 remain */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity(knex, 'E1', 'E2', 'E3', 'E4'); await insertReference( knex, @@ -318,6 +303,6 @@ describe('deleteWithEagerPruningOfChildren', () => { 'E4', ]); await expect(entitiesMarkedForStitching(knex)).resolves.toEqual([]); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts b/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts index e37f764ce0..dc7da9f7e3 100644 --- a/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts +++ b/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { randomUUID as uuid } from 'node:crypto'; import { applyDatabaseMigrations } from '../../migrations'; import { DbRefreshKeysRow, DbRefreshStateRow } from '../../tables'; @@ -23,19 +23,19 @@ import { refreshByRefreshKeys } from './refreshByRefreshKeys'; jest.setTimeout(60_000); -describe('refreshByRefreshKeys', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return knex; - } +describe.each(databases.eachSupportedId())( + 'refreshByRefreshKeys, %p', + databaseId => { + async function createDatabase() { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return knex; + } - it.each(databases.eachSupportedId())( - 'works for the simple path, %p', - async databaseId => { - const knex = await createDatabase(databaseId); + it('works for the simple path', async () => { + const knex = await createDatabase(); const eid1 = uuid(); await knex('refresh_state').insert({ @@ -89,6 +89,6 @@ describe('refreshByRefreshKeys', () => { expect(normalizeTimestamp(before2.next_update_at)).toEqual( normalizeTimestamp(after2.next_update_at), ); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts index 1f6137f91e..f78ae8d89d 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/getDeferredStitchableEntities.test.ts @@ -21,12 +21,12 @@ import { getDeferredStitchableEntities } from './getDeferredStitchableEntities'; jest.setTimeout(60_000); -describe('getDeferredStitchableEntities', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'selects the right rows %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'getDeferredStitchableEntities, %p', + databaseId => { + it('selects the right rows', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -83,6 +83,6 @@ describe('getDeferredStitchableEntities', () => { expect(+new Date(hitRowAfter!)).toBeGreaterThan(+new Date(hitRowBefore!)); expect(+new Date(missRowAfter!)).toEqual(+new Date(missRowBefore!)); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts index dbcbba13df..ca0ac590b4 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markDeferredStitchCompleted.test.ts @@ -21,12 +21,12 @@ import { DbStitchQueueRow } from '../../tables'; jest.setTimeout(60_000); -describe('markDeferredStitchCompleted', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'completes only if unchanged %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'markDeferredStitchCompleted, %p', + databaseId => { + it('completes only if unchanged', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -68,6 +68,6 @@ describe('markDeferredStitchCompleted', () => { stitchTicket: 'the-ticket', }); await expect(result()).resolves.toEqual([]); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts index f7fd9a0695..c9a89f8892 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts @@ -25,12 +25,12 @@ import { jest.setTimeout(60_000); -describe('markForStitching', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'marks the right rows in deferred mode %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'markForStitching, %p', + databaseId => { + it('marks the right rows in deferred mode', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -203,12 +203,9 @@ describe('markForStitching', () => { const final = await result(); const entity4Final = final.find(r => r.entity_ref === 'k:ns/four'); expect(entity4Final?.stitch_ticket).not.toEqual('old'); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'marks the right rows in immediate mode %p', - async databaseId => { + it('marks the right rows in immediate mode', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -435,12 +432,9 @@ describe('markForStitching', () => { for (let i = 0; i < final.length; ++i) { expect(original[i].next_update_at).not.toEqual(final[i].next_update_at); } - }, - ); + }); - it.each(databases.eachSupportedId())( - 'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p', - async databaseId => { + it('reproduces deadlock scenario when concurrent transactions update overlapping entity sets', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -567,6 +561,6 @@ describe('markForStitching', () => { expect(row.next_stitch_at).not.toBeNull(); expect(row.stitch_ticket).not.toBeNull(); }); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts index 41931a8b7e..6099ee031b 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/performStitching.test.ts @@ -29,14 +29,15 @@ import { performStitching } from './performStitching'; jest.setTimeout(60_000); -describe('performStitching', () => { - const databases = TestDatabases.create(); - const logger = mockServices.logger.mock(); +const databases = TestDatabases.create(); - // NOTE(freben): Testing the deferred path since it's a superset of the immediate one - it.each(databases.eachSupportedId())( - 'runs the happy path in deferred mode for %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'performStitching, %p', + databaseId => { + const logger = mockServices.logger.mock(); + + // NOTE(freben): Testing the deferred path since it's a superset of the immediate one + it('runs the happy path in deferred mode', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -323,12 +324,9 @@ describe('performStitching', () => { }, ]), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'handles conflicts with past stitches %p', - async databaseId => { + it('handles conflicts with past stitches', async () => { if (databaseId === 'MYSQL_8') { // MySQL doesn't handle conflicts in the same way as the other two, most // likely due to the conflict probably being handled with a merged even @@ -391,12 +389,9 @@ describe('performStitching', () => { 'Skipping stitching of k:ns/n, conflict', expect.anything(), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'stitches when final_entities row already exists %p', - async databaseId => { + it('stitches when final_entities row already exists', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -444,6 +439,6 @@ describe('performStitching', () => { expect(entities.length).toBe(1); expect(entities[0].hash).not.toBe(''); expect(entities[0].final_entity).toBeDefined(); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts index c61c425a30..b14367cd12 100644 --- a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts +++ b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { Knex } from 'knex'; import { StitchingStrategy } from '../../../stitching/types'; import { applyDatabaseMigrations } from '../../migrations'; @@ -28,109 +28,112 @@ import { deleteOrphanedEntities } from './deleteOrphanedEntities'; jest.setTimeout(60_000); -describe('deleteOrphanedEntities', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return knex; - } +describe.each(databases.eachSupportedId())( + 'deleteOrphanedEntities, %p', + databaseId => { + async function createDatabase() { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return knex; + } - async function run(knex: Knex, strategy: StitchingStrategy): Promise { - let result: number; - await knex.transaction( - async tx => { - // We can't return here, as knex swallows the return type in case the - // transaction is rolled back: - // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 - result = await deleteOrphanedEntities({ knex: tx, strategy }); - }, - { - // If we explicitly trigger a rollback, don't fail. - doNotRejectOnRollback: true, - }, - ); - return result!; - } + async function run( + knex: Knex, + strategy: StitchingStrategy, + ): Promise { + let result: number; + await knex.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the + // transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await deleteOrphanedEntities({ knex: tx, strategy }); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + return result!; + } - async function insertEntity(knex: Knex, ...entityRefs: string[]) { - for (const ref of entityRefs) { - await knex('refresh_state').insert({ - entity_id: `id-${ref}`, - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - result_hash: 'original', - }); - await knex('final_entities').insert({ - entity_id: `id-${ref}`, - hash: 'original', - entity_ref: ref, + async function insertEntity(knex: Knex, ...entityRefs: string[]) { + for (const ref of entityRefs) { + await knex('refresh_state').insert({ + entity_id: `id-${ref}`, + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + result_hash: 'original', + }); + await knex('final_entities').insert({ + entity_id: `id-${ref}`, + hash: 'original', + entity_ref: ref, + }); + } + } + + async function insertReference( + knex: Knex, + ...refs: DbRefreshStateReferencesRow[] + ) { + await knex( + 'refresh_state_references', + ).insert(refs); + } + + async function insertRelation(knex: Knex, fromRef: string, toRef: string) { + const orig = await knex + .select('entity_id') + .from('refresh_state') + .where('entity_ref', fromRef); + await knex('relations').insert({ + originating_entity_id: orig[0].entity_id, + type: 'fake', + source_entity_ref: fromRef, + target_entity_ref: toRef, }); } - } - async function insertReference( - knex: Knex, - ...refs: DbRefreshStateReferencesRow[] - ) { - await knex('refresh_state_references').insert( - refs, - ); - } + async function refreshState(knex: Knex) { + return await knex('refresh_state') + .orderBy('entity_ref') + .select('entity_ref', 'result_hash'); + } - async function insertRelation(knex: Knex, fromRef: string, toRef: string) { - const orig = await knex - .select('entity_id') - .from('refresh_state') - .where('entity_ref', fromRef); - await knex('relations').insert({ - originating_entity_id: orig[0].entity_id, - type: 'fake', - source_entity_ref: fromRef, - target_entity_ref: toRef, - }); - } + async function stitchQueue(knex: Knex) { + return await knex('stitch_queue') + .orderBy('entity_ref') + .select('entity_ref'); + } - async function refreshState(knex: Knex) { - return await knex('refresh_state') - .orderBy('entity_ref') - .select('entity_ref', 'result_hash'); - } + async function finalEntities(knex: Knex) { + return await knex('final_entities') + .join( + 'refresh_state', + 'final_entities.entity_id', + 'refresh_state.entity_id', + ) + .leftOuterJoin( + 'stitch_queue', + 'stitch_queue.entity_ref', + 'refresh_state.entity_ref', + ) + .orderBy('refresh_state.entity_ref') + .select({ + entity_ref: 'refresh_state.entity_ref', + hash: 'final_entities.hash', + next_stitch_at: 'stitch_queue.next_stitch_at', + }); + } - async function stitchQueue(knex: Knex) { - return await knex('stitch_queue') - .orderBy('entity_ref') - .select('entity_ref'); - } - - async function finalEntities(knex: Knex) { - return await knex('final_entities') - .join( - 'refresh_state', - 'final_entities.entity_id', - 'refresh_state.entity_id', - ) - .leftOuterJoin( - 'stitch_queue', - 'stitch_queue.entity_ref', - 'refresh_state.entity_ref', - ) - .orderBy('refresh_state.entity_ref') - .select({ - entity_ref: 'refresh_state.entity_ref', - hash: 'final_entities.hash', - next_stitch_at: 'stitch_queue.next_stitch_at', - }); - } - - it.each(databases.eachSupportedId())( - 'works for some mixed paths in immediate mode, %p', - async databaseId => { + it('works for some mixed paths in immediate mode', async () => { /* In this graph, edges represent refresh state references, not entity relations: @@ -155,7 +158,7 @@ describe('deleteOrphanedEntities', () => { Result: E3, E4, E5, E6, and E10 deleted; others remain Entities that had relations pointing at orphans are marked for reprocessing */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity( knex, 'E1', @@ -210,12 +213,9 @@ describe('deleteOrphanedEntities', () => { { entity_ref: 'E8', hash: 'original', next_stitch_at: null }, { entity_ref: 'E9', hash: 'original', next_stitch_at: null }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'works for some mixed paths in deferred mode, %p', - async databaseId => { + it('works for some mixed paths in deferred mode', async () => { /* In this graph, edges represent refresh state references, not entity relations: @@ -240,7 +240,7 @@ describe('deleteOrphanedEntities', () => { Result: E3, E4, E5, E6, and E10 deleted; others remain Entities that had relations pointing at orphans are marked for reprocessing */ - const knex = await createDatabase(databaseId); + const knex = await createDatabase(); await insertEntity( knex, 'E1', @@ -304,6 +304,6 @@ describe('deleteOrphanedEntities', () => { { entity_ref: 'E8', hash: 'original', next_stitch_at: null }, { entity_ref: 'E9', hash: 'original', next_stitch_at: null }, ]); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts index 91b6f7b29f..5a39dbd19a 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { ANNOTATION_ORIGIN_LOCATION, stringifyEntityRef, @@ -37,63 +37,58 @@ import waitFor from 'wait-for-expect'; jest.setTimeout(60_000); -describe('DefaultLocationStore', () => { - const databases = TestDatabases.create(); - const mockScmEvents = { - subscribe: jest.fn(), - publish: jest.fn(), - markEventActionTaken: jest.fn(), - }; - let subscriber: CatalogScmEventsServiceSubscriber | undefined; +const databases = TestDatabases.create(); - beforeEach(() => { - jest.clearAllMocks(); +describe.each(databases.eachSupportedId())( + 'DefaultLocationStore, %p', + databaseId => { + const mockScmEvents = { + subscribe: jest.fn(), + publish: jest.fn(), + markEventActionTaken: jest.fn(), + }; + let subscriber: CatalogScmEventsServiceSubscriber | undefined; - subscriber = undefined; - mockScmEvents.subscribe.mockImplementation(sub => { - subscriber = sub; - return { unsubscribe: () => {} }; + beforeEach(() => { + jest.clearAllMocks(); + + subscriber = undefined; + mockScmEvents.subscribe.mockImplementation(sub => { + subscriber = sub; + return { unsubscribe: () => {} }; + }); }); - }); - async function createLocationStore(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - const connection = { applyMutation: jest.fn(), refresh: jest.fn() }; - const store = new DefaultLocationStore(knex, mockScmEvents, { - refresh: true, - unregister: true, - move: true, - }); - await store.connect(connection); - return { store, connection, knex }; - } + async function createLocationStore() { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + const connection = { applyMutation: jest.fn(), refresh: jest.fn() }; + const store = new DefaultLocationStore(knex, mockScmEvents, { + refresh: true, + unregister: true, + move: true, + }); + await store.connect(connection); + return { store, connection, knex }; + } - it.each(databases.eachSupportedId())( - 'should do a full sync with the locations on connect, %p', - async databaseId => { - const { connection } = await createLocationStore(databaseId); + it('should do a full sync with the locations on connect', async () => { + const { connection } = await createLocationStore(); expect(connection.applyMutation).toHaveBeenCalledWith({ type: 'full', entities: [], }); - }, - ); + }); - describe('listLocations', () => { - it.each(databases.eachSupportedId())( - 'lists empty locations when there is no locations, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + describe('listLocations', () => { + it('lists empty locations when there is no locations', async () => { + const { store } = await createLocationStore(); expect(await store.listLocations()).toEqual([]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'lists locations that are added to the db, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + it('lists locations that are added to the db', async () => { + const { store } = await createLocationStore(); await store.createLocation({ target: 'https://github.com/backstage/demo/blob/master/catalog-info.yml', @@ -111,15 +106,12 @@ describe('DefaultLocationStore', () => { }), ]), ); - }, - ); - }); + }); + }); - describe('createLocation', () => { - it.each(databases.eachSupportedId())( - 'throws when the location already exists, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + describe('createLocation', () => { + it('throws when the location already exists', async () => { + const { store } = await createLocationStore(); const spec = { target: 'https://github.com/backstage/demo/blob/master/catalog-info.yml', @@ -129,13 +121,10 @@ describe('DefaultLocationStore', () => { await expect(() => store.createLocation(spec)).rejects.toThrow( new RegExp(`Location ${spec.type}:${spec.target} already exists`), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'calls apply mutation when adding a new location, %p', - async databaseId => { - const { store, connection } = await createLocationStore(databaseId); + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); await store.createLocation({ target: 'https://github.com/backstage/demo/blob/master/catalog-info.yml', @@ -159,13 +148,10 @@ describe('DefaultLocationStore', () => { }, ]), }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'updates refresh_state when onConflict is refresh, %p', - async databaseId => { - const { store, knex } = await createLocationStore(databaseId); + it('updates refresh_state when onConflict is refresh', async () => { + const { store, knex } = await createLocationStore(); const spec = { type: 'url', target: @@ -201,13 +187,10 @@ describe('DefaultLocationStore', () => { expect(new Date(row.next_update_at).getTime()).toBeGreaterThan( oldDate.getTime(), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'persists the correct location_entity_ref when creating a location, %p', - async databaseId => { - const { store, knex } = await createLocationStore(databaseId); + it('persists the correct location_entity_ref when creating a location', async () => { + const { store, knex } = await createLocationStore(); const created = await store.createLocation({ type: 'url', target: @@ -222,26 +205,20 @@ describe('DefaultLocationStore', () => { expect(row.location_entity_ref).toBe( 'location:default/generated-fa35d9c166e43ab7f4a7c59a00e88e4e8b5aba34', ); - }, - ); - }); + }); + }); - describe('deleteLocation', () => { - it.each(databases.eachSupportedId())( - 'throws if the location does not exist, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + describe('deleteLocation', () => { + it('throws if the location does not exist', async () => { + const { store } = await createLocationStore(); const id = uuid(); await expect(() => store.deleteLocation(id)).rejects.toThrow( new RegExp(`Found no location with ID ${id}`), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'calls apply mutation when adding a new location, %p', - async databaseId => { - const { store, connection } = await createLocationStore(databaseId); + it('calls apply mutation when adding a new location', async () => { + const { store, connection } = await createLocationStore(); const location = await store.createLocation({ target: @@ -268,15 +245,12 @@ describe('DefaultLocationStore', () => { }, ], }); - }, - ); - }); + }); + }); - describe('updateLocation', () => { - it.each(databases.eachSupportedId())( - 'throws if the location does not exist, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + describe('updateLocation', () => { + it('throws if the location does not exist', async () => { + const { store } = await createLocationStore(); const id = uuid(); await expect(() => store.updateLocation(id, { @@ -284,13 +258,10 @@ describe('DefaultLocationStore', () => { target: 'https://example.com', }), ).rejects.toThrow(new RegExp(`Found no location with ID ${id}`)); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'throws ConflictError when updating to a type+target already used by another location, %p', - async databaseId => { - const { store } = await createLocationStore(databaseId); + it('throws ConflictError when updating to a type+target already used by another location', async () => { + const { store } = await createLocationStore(); await store.createLocation({ type: 'url', @@ -307,13 +278,10 @@ describe('DefaultLocationStore', () => { target: 'https://example.com/a', }), ).rejects.toThrow(/already exists/); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'updates type and target and issues a delta mutation with the new entity, %p', - async databaseId => { - const { store, connection } = await createLocationStore(databaseId); + it('updates type and target and issues a delta mutation with the new entity', async () => { + const { store, connection } = await createLocationStore(); const created = await store.createLocation({ type: 'url', @@ -345,15 +313,12 @@ describe('DefaultLocationStore', () => { }, ], }); - }, - ); - }); + }); + }); - describe('getLocationByEntity', () => { - it.each(databases.eachSupportedId())( - 'loads correctly, %p', - async databaseId => { - const { store, knex } = await createLocationStore(databaseId); + describe('getLocationByEntity', () => { + it('loads correctly', async () => { + const { store, knex } = await createLocationStore(); const entityId = uuid(); const locationId = uuid(); @@ -407,16 +372,12 @@ describe('DefaultLocationStore', () => { ).rejects.toMatchInlineSnapshot( `[NotFoundError: found no entity for ref k:ns/n2]`, ); - }, - ); - }); + }); + }); - describe('SCM event handling', () => { - describe.each(databases.eachSupportedId())('%p', databaseId => { + describe('SCM event handling', () => { it('handles location.deleted', async () => { - const { store, knex, connection } = await createLocationStore( - databaseId, - ); + const { store, knex, connection } = await createLocationStore(); expect(subscriber).not.toBeUndefined(); // Prepare @@ -531,9 +492,7 @@ describe('DefaultLocationStore', () => { }); it('handles location.moved', async () => { - const { store, knex, connection } = await createLocationStore( - databaseId, - ); + const { store, knex, connection } = await createLocationStore(); expect(subscriber).not.toBeUndefined(); // Prepare @@ -669,9 +628,7 @@ describe('DefaultLocationStore', () => { }); it('handles repository.deleted', async () => { - const { store, knex, connection } = await createLocationStore( - databaseId, - ); + const { store, knex, connection } = await createLocationStore(); expect(subscriber).not.toBeUndefined(); // Prepare @@ -787,9 +744,7 @@ describe('DefaultLocationStore', () => { }); it('handles repository.moved', async () => { - const { store, knex, connection } = await createLocationStore( - databaseId, - ); + const { store, knex, connection } = await createLocationStore(); expect(subscriber).not.toBeUndefined(); // Prepare @@ -919,45 +874,42 @@ describe('DefaultLocationStore', () => { }); }); }); - }); - describe('queryLocations', () => { - const l1 = { - id: '00000000-0000-0000-0000-000000000001', - type: 'url', - target: - 'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml', - entityRef: - 'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a', - }; - const l2 = { - id: '00000000-0000-0000-0000-000000000002', - type: 'url', - target: - 'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml', - entityRef: - 'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce', - }; - const l3 = { - id: '00000000-0000-0000-0000-000000000003', - type: 'url', - target: - 'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml', - entityRef: - 'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e', - }; - const l4 = { - id: '00000000-0000-0000-0000-000000000004', - type: 'file', - target: '/tmp/catalog-info.yaml', - entityRef: - 'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2', - }; + describe('queryLocations', () => { + const l1 = { + id: '00000000-0000-0000-0000-000000000001', + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/packages/catalog-model/catalog-info.yaml', + entityRef: + 'location:default/generated-0ecbc46527aae891650cc1ad4eb17e15391fa96a', + }; + const l2 = { + id: '00000000-0000-0000-0000-000000000002', + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/plugins/catalog/catalog-info.yaml', + entityRef: + 'location:default/generated-888dd2d9775aaf5b722ebdece23c21e2541e90ce', + }; + const l3 = { + id: '00000000-0000-0000-0000-000000000003', + type: 'url', + target: + 'https://github.com/backstage/backstage/blob/master/plugins/scaffolder/catalog-info.yaml', + entityRef: + 'location:default/generated-d4255ab29a8321cb6eae30cee45969a272e1206e', + }; + const l4 = { + id: '00000000-0000-0000-0000-000000000004', + type: 'file', + target: '/tmp/catalog-info.yaml', + entityRef: + 'location:default/generated-d14ac9f97f7d042d45b2130dcf3d087e000f07f2', + }; - it.each(databases.eachSupportedId())( - 'queries locations correctly, %p', - async databaseId => { - const { store, knex } = await createLocationStore(databaseId); + it('queries locations correctly', async () => { + const { store, knex } = await createLocationStore(); // Insert locations in a random order to test the sorting const locations = [l1, l2, l3, l4]; @@ -1195,7 +1147,7 @@ describe('DefaultLocationStore', () => { items: [], totalItems: 0, }); - }, - ); - }); -}); + }); + }); + }, +); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 80637e6bae..3836a1d30a 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -15,7 +15,6 @@ */ import { - TestDatabaseId, TestDatabases, mockCredentials, mockServices, @@ -42,102 +41,103 @@ import { entitiesResponseToObjects } from './response'; jest.setTimeout(60_000); -describe('DefaultEntitiesCatalog', () => { - let knex: Knex; +const databases = TestDatabases.create(); - afterEach(async () => { - await knex.destroy(); - }); +describe.each(databases.eachSupportedId())( + 'DefaultEntitiesCatalog, %p', + databaseId => { + let knex: Knex; - const databases = TestDatabases.create(); - const stitch = jest.fn(); - const stitcher: Stitcher = { stitch } as any; - - async function createDatabase(databaseId: TestDatabaseId) { - knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - } - - async function addEntity( - entity: Entity, - parents: { source?: string; entity?: Entity }[], - ) { - const id = uuid(); - const entityRef = stringifyEntityRef(entity); - const entityJson = JSON.stringify(entity); - - await knex('refresh_state').insert({ - entity_id: id, - entity_ref: entityRef, - unprocessed_entity: entityJson, - errors: '[]', - next_update_at: '2031-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', + afterEach(async () => { + await knex.destroy(); }); - await knex('final_entities').insert({ - entity_id: id, - entity_ref: entityRef, - final_entity: entityJson, - hash: 'h', - }); + const stitch = jest.fn(); + const stitcher: Stitcher = { stitch } as any; - for (const parent of parents) { - await knex( - 'refresh_state_references', - ).insert({ - source_key: parent.source, - source_entity_ref: parent.entity && stringifyEntityRef(parent.entity), - target_entity_ref: stringifyEntityRef(entity), - }); + async function createDatabase() { + knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); } - const search = await buildEntitySearch(id, entity); - await knex('search').insert(search); + async function addEntity( + entity: Entity, + parents: { source?: string; entity?: Entity }[], + ) { + const id = uuid(); + const entityRef = stringifyEntityRef(entity); + const entityJson = JSON.stringify(entity); - return id; - } - - async function addEntityToSearch(entity: Entity) { - const id = entity.metadata.uid || uuid(); - const entityRef = stringifyEntityRef(entity); - const entityJson = JSON.stringify(entity); - - await knex('refresh_state').insert({ - entity_id: id, - entity_ref: entityRef, - unprocessed_entity: entityJson, - errors: '[]', - next_update_at: '2031-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - - await knex('final_entities').insert({ - entity_id: id, - entity_ref: entityRef, - final_entity: entityJson, - hash: 'h', - }); - - for (const row of buildEntitySearch(id, entity)) { - await knex('search').insert({ + await knex('refresh_state').insert({ entity_id: id, - key: row.key, - value: row.value, - original_value: row.original_value, + entity_ref: entityRef, + unprocessed_entity: entityJson, + errors: '[]', + next_update_at: '2031-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', }); + + await knex('final_entities').insert({ + entity_id: id, + entity_ref: entityRef, + final_entity: entityJson, + hash: 'h', + }); + + for (const parent of parents) { + await knex( + 'refresh_state_references', + ).insert({ + source_key: parent.source, + source_entity_ref: parent.entity && stringifyEntityRef(parent.entity), + target_entity_ref: stringifyEntityRef(entity), + }); + } + + const search = await buildEntitySearch(id, entity); + await knex('search').insert(search); + + return id; } - } - afterEach(() => { - jest.resetAllMocks(); - }); + async function addEntityToSearch(entity: Entity) { + const id = entity.metadata.uid || uuid(); + const entityRef = stringifyEntityRef(entity); + const entityJson = JSON.stringify(entity); - describe('entityAncestry', () => { - it.each(databases.eachSupportedId())( - 'should return the ancestry with one parent, %p', - async databaseId => { - await createDatabase(databaseId); + await knex('refresh_state').insert({ + entity_id: id, + entity_ref: entityRef, + unprocessed_entity: entityJson, + errors: '[]', + next_update_at: '2031-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + await knex('final_entities').insert({ + entity_id: id, + entity_ref: entityRef, + final_entity: entityJson, + hash: 'h', + }); + + for (const row of buildEntitySearch(id, entity)) { + await knex('search').insert({ + entity_id: id, + key: row.key, + value: row.value, + original_value: row.original_value, + }); + } + } + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('entityAncestry', () => { + it('should return the ancestry with one parent', async () => { + await createDatabase(); const grandparent: Entity = { apiVersion: 'a', @@ -188,13 +188,10 @@ describe('DefaultEntitiesCatalog', () => { }, ]), ); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should throw error if the entity does not exist, %p', - async databaseId => { - await createDatabase(databaseId); + it('should throw error if the entity does not exist', async () => { + await createDatabase(); const catalog = new DefaultEntitiesCatalog({ database: knex, logger: mockServices.logger.mock(), @@ -203,13 +200,10 @@ describe('DefaultEntitiesCatalog', () => { await expect(() => catalog.entityAncestry('k:default/root'), ).rejects.toThrow('No such entity k:default/root'); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return the ancestry with multiple parents, %p', - async databaseId => { - await createDatabase(databaseId); + it('should return the ancestry with multiple parents', async () => { + await createDatabase(); const grandparent: Entity = { apiVersion: 'a', @@ -275,15 +269,12 @@ describe('DefaultEntitiesCatalog', () => { }, ]), ); - }, - ); - }); + }); + }); - describe('entities', () => { - it.each(databases.eachSupportedId())( - 'should return correct entity for simple filter, %p', - async databaseId => { - await createDatabase(databaseId); + describe('entities', () => { + it('should return correct entity for simple filter', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -317,13 +308,10 @@ describe('DefaultEntitiesCatalog', () => { expect(entities.length).toBe(1); expect(entities[0]).toEqual(entity2); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return correct entity for negation filter, %p', - async databaseId => { - await createDatabase(databaseId); + it('should return correct entity for negation filter', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -359,13 +347,10 @@ describe('DefaultEntitiesCatalog', () => { expect(entities.length).toBe(1); expect(entities[0]).toEqual(entity1); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return correct entities for nested filter, %p', - async databaseId => { - await createDatabase(databaseId); + it('should return correct entities for nested filter', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -433,13 +418,10 @@ describe('DefaultEntitiesCatalog', () => { expect(entities.length).toBe(2); expect(entities).toContainEqual(entity2); expect(entities).toContainEqual(entity4); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return correct entities for complex negation filter, %p', - async databaseId => { - await createDatabase(databaseId); + it('should return correct entities for complex negation filter', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -480,14 +462,11 @@ describe('DefaultEntitiesCatalog', () => { expect(entities.length).toBe(1); expect(entities).toContainEqual(entity1); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return no matches for an empty values array, %p', - // NOTE: An empty values array is not a sensible input in a realistic scenario. - async databaseId => { - await createDatabase(databaseId); + it('should return no matches for an empty values array', async () => { + // NOTE: An empty values array is not a sensible input in a realistic scenario. + await createDatabase(); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -519,13 +498,10 @@ describe('DefaultEntitiesCatalog', () => { const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(0); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return both target and targetRef for entities in compat mode', - async databaseId => { - await createDatabase(databaseId); + it('should return both target and targetRef for entities in compat mode', async () => { + await createDatabase(); await addEntity( { apiVersion: 'a', @@ -579,13 +555,10 @@ describe('DefaultEntitiesCatalog', () => { target: { kind: 'x', namespace: 'y', name: 'z' }, }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'handles inversion both for existing and missing keys, %p', - async databaseId => { - await createDatabase(databaseId); + it('handles inversion both for existing and missing keys', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', @@ -638,13 +611,10 @@ describe('DefaultEntitiesCatalog', () => { filter: { not: { key: 'spec.b', values: ['lonely'] } }, }), ).resolves.toEqual(['n1', 'n3']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can order and combine with filtering, %p', - async databaseId => { - await createDatabase(databaseId); + it('can order and combine with filtering', async () => { + await createDatabase(); const entity1: Entity = { apiVersion: 'a', @@ -739,13 +709,10 @@ describe('DefaultEntitiesCatalog', () => { ], }), ).resolves.toEqual(['n4', 'n3', 'n1', 'n2']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'paginates correctly through single-field ordering, %p', - async databaseId => { - await createDatabase(databaseId); + it('paginates correctly through single-field ordering', async () => { + await createDatabase(); // All four entities have metadata.name — fast path uses Phase 1 only for (const name of ['n1', 'n2', 'n3', 'n4']) { @@ -795,13 +762,10 @@ describe('DefaultEntitiesCatalog', () => { expect(await hasNext(2, 1)).toBe(true); await expect(page(100)).resolves.toEqual(['n1', 'n2', 'n3', 'n4']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'paginates across the Phase 1 / Phase 2 boundary, %p', - async databaseId => { - await createDatabase(databaseId); + it('paginates across the Phase 1 / Phase 2 boundary', async () => { + await createDatabase(); // n1 and n2 have spec.b (Phase 1); n3 and n4 do not (Phase 2). // Explicit UIDs pin Phase 2 ordering (entity_id ASC) to a known sequence. @@ -867,13 +831,10 @@ describe('DefaultEntitiesCatalog', () => { 'n3', 'n4', ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'treats a null sort-field value the same as a missing sort field, %p', - async databaseId => { - await createDatabase(databaseId); + it('treats a null sort-field value the same as a missing sort field', async () => { + await createDatabase(); // n1 has spec.b with a real value (Phase 1) // n2 has spec.b explicitly set to null — buildEntitySearch stores value=NULL @@ -918,15 +879,12 @@ describe('DefaultEntitiesCatalog', () => { // ordered by entity_id ASC, regardless of primary direction await expect(page('asc')).resolves.toEqual(['n1', 'n2', 'n3']); await expect(page('desc')).resolves.toEqual(['n1', 'n2', 'n3']); - }, - ); - }); + }); + }); - describe('entitiesBatch', () => { - it.each(databases.eachSupportedId())( - 'queries for entities by ref, including duplicates, and gracefully returns null for missing entities, %p', - async databaseId => { - await createDatabase(databaseId); + describe('entitiesBatch', () => { + it('queries for entities by ref, including duplicates, and gracefully returns null for missing entities', async () => { + await createDatabase(); await addEntity( { @@ -976,13 +934,10 @@ describe('DefaultEntitiesCatalog', () => { null, 'k:default/two', ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'queries for entities by ref, including filtering, %p', - async databaseId => { - await createDatabase(databaseId); + it('queries for entities by ref, including filtering', async () => { + await createDatabase(); await addEntity( { @@ -1022,15 +977,12 @@ describe('DefaultEntitiesCatalog', () => { 'k:default/two', null, ]); - }, - ); - }); + }); + }); - describe('queryEntities', () => { - it.each(databases.eachSupportedId())( - 'should return paginated entities and scroll the items accordingly, %p', - async databaseId => { - await createDatabase(databaseId); + describe('queryEntities', () => { + it('should return paginated entities and scroll the items accordingly', async () => { + await createDatabase(); const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; const entities: Entity[] = names.map(name => entityFrom(name)); @@ -1201,13 +1153,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response8.pageInfo.nextCursor).toBeUndefined(); expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should return paginated entities ordered in descending order and scroll the items accordingly, %p', - async databaseId => { - await createDatabase(databaseId); + it('should return paginated entities ordered in descending order and scroll the items accordingly', async () => { + await createDatabase(); const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; const entities: Entity[] = names.map(name => entityFrom(name)); @@ -1379,13 +1328,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response8.pageInfo.nextCursor).toBeUndefined(); expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should filter the results when query is provided, %p', - async databaseId => { - await createDatabase(databaseId); + it('should filter the results when query is provided', async () => { + await createDatabase(); const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; const entities: Entity[] = names.map(name => entityFrom(name)); @@ -1435,13 +1381,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response.pageInfo.nextCursor).toBeUndefined(); expect(response.pageInfo.prevCursor).toBeUndefined(); expect(response.totalItems).toBe(3); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should filter the results when query is provided with fullTextFilter for camelCase fields, %p', - async databaseId => { - await createDatabase(databaseId); + it('should filter the results when query is provided with fullTextFilter for camelCase fields', async () => { + await createDatabase(); const entities: Entity[] = [ { @@ -1489,13 +1432,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response.pageInfo.nextCursor).toBeUndefined(); expect(response.pageInfo.prevCursor).toBeUndefined(); expect(response.totalItems).toBe(1); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should filter the text results when sortOrder is not provided, %p', - async databaseId => { - await createDatabase(databaseId); + it('should filter the text results when sortOrder is not provided', async () => { + await createDatabase(); const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; const entities: Entity[] = names.map((name, index) => @@ -1577,13 +1517,10 @@ describe('DefaultEntitiesCatalog', () => { credentials: mockCredentials.none(), }); expect(paginatedResponsePrev).toMatchObject(paginatedResponse); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should filter the text results by multiple search fields if provided, %p', - async databaseId => { - await createDatabase(databaseId); + it('should filter the text results by multiple search fields if provided', async () => { + await createDatabase(); const defs = [ { @@ -1682,13 +1619,10 @@ describe('DefaultEntitiesCatalog', () => { credentials: mockCredentials.none(), }); expect(paginatedResponsePrev).toMatchObject(paginatedResponse); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should include totalItems and empty entities in the response in case limit is zero, %p', - async databaseId => { - await createDatabase(databaseId); + it('should include totalItems and empty entities in the response in case limit is zero', async () => { + await createDatabase(); await Promise.all( Array(20) @@ -1718,13 +1652,10 @@ describe('DefaultEntitiesCatalog', () => { items: { type: 'raw', entities: [] }, pageInfo: {}, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can skip totalItems, %p', - async databaseId => { - await createDatabase(databaseId); + it('can skip totalItems', async () => { + await createDatabase(); await Promise.all( Array(15) @@ -1772,13 +1703,10 @@ describe('DefaultEntitiesCatalog', () => { pageInfo: { prevCursor: expect.anything() }, }); expect(response.items.entities).toHaveLength(5); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should paginate results accordingly in case of clashing items, %p', - async databaseId => { - await createDatabase(databaseId); + it('should paginate results accordingly in case of clashing items', async () => { + await createDatabase(); await Promise.all([ addEntityToSearch(entityFrom('AA')), @@ -1871,13 +1799,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.pageInfo.nextCursor).toBeDefined(); expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(6); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should exclude filtered entities when paginating, %p', - async databaseId => { - await createDatabase(databaseId); + it('should exclude filtered entities when paginating', async () => { + await createDatabase(); await Promise.all([ addEntityToSearch(entityFrom('AA', { uid: '1', kind: 'included' })), @@ -1954,13 +1879,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response2.pageInfo.nextCursor).toBeDefined(); expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(6); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should paginate results without sort fields, %p', - async databaseId => { - await createDatabase(databaseId); + it('should paginate results without sort fields', async () => { + await createDatabase(); await Promise.all([ addEntityToSearch(entityFrom('AA', { uid: 'id1' })), @@ -2058,13 +1980,10 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.pageInfo.nextCursor).toBeDefined(); expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(6); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should sort properly for fields that do not exist on all entities, %p', - async databaseId => { - await createDatabase(databaseId); + it('should sort properly for fields that do not exist on all entities', async () => { + await createDatabase(); await Promise.all([ addEntityToSearch(entityFrom('AA', { uid: 'id1' })), @@ -2100,13 +2019,10 @@ describe('DefaultEntitiesCatalog', () => { ), ).toEqual(['BB', 'CC']); expect(descResult.totalItems).toBe(2); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should silently skip over entities that are not yet stitched, %p', - async databaseId => { - await createDatabase(databaseId); + it('should silently skip over entities that are not yet stitched', async () => { + await createDatabase(); const entity1 = entityFrom('AA', { uid: 'id1' }); const entity2 = entityFrom('BB', { uid: 'id2' }); @@ -2149,13 +2065,10 @@ describe('DefaultEntitiesCatalog', () => { entitiesResponseToObjects(r.items).map(e => e!.metadata.name), ), ).resolves.toEqual(['BB']); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should not return duplicate entities when using orderField, %p', - async databaseId => { - await createDatabase(databaseId); + it('should not return duplicate entities when using orderField', async () => { + await createDatabase(); // Create a few test entities with different names to sort by const entities = [ @@ -2234,13 +2147,10 @@ describe('DefaultEntitiesCatalog', () => { 'b-entity', 'c-entity', ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should apply both filter and query when both are given, %p', - async databaseId => { - await createDatabase(databaseId); + it('should apply both filter and query when both are given', async () => { + await createDatabase(); // Add entities with different kinds and names await addEntityToSearch(entityFrom('A', { kind: 'component' })); @@ -2266,15 +2176,12 @@ describe('DefaultEntitiesCatalog', () => { expect(resultEntities).toEqual([ entityFrom('A', { kind: 'component' }), ]); - }, - ); - }); + }); + }); - describe('removeEntityByUid', () => { - it.each(databases.eachSupportedId())( - 'also clears parent hashes, %p', - async databaseId => { - await createDatabase(databaseId); + describe('removeEntityByUid', () => { + it('also clears parent hashes', async () => { + await createDatabase(); const grandparent: Entity = { apiVersion: 'a', @@ -2358,15 +2265,12 @@ describe('DefaultEntitiesCatalog', () => { expect(stitch).toHaveBeenCalledWith({ entityRefs: new Set(['k:default/unrelated1', 'k:default/unrelated2']), }); - }, - ); - }); + }); + }); - describe('facets', () => { - it.each(databases.eachSupportedId())( - 'can filter and collect properly, %p', - async databaseId => { - await createDatabase(databaseId); + describe('facets', () => { + it('can filter and collect properly', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2432,13 +2336,10 @@ describe('DefaultEntitiesCatalog', () => { kind: [{ value: 'k', count: 1 }], }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can match on annotations and labels with dots in them, %p', - async databaseId => { - await createDatabase(databaseId); + it('can match on annotations and labels with dots in them', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2483,13 +2384,10 @@ describe('DefaultEntitiesCatalog', () => { ], }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'can match on strings in arrays, %p', - async databaseId => { - await createDatabase(databaseId); + it('can match on strings in arrays', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2529,13 +2427,10 @@ describe('DefaultEntitiesCatalog', () => { ]), }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'works with a mixture of present and missing facets, %p', - async databaseId => { - await createDatabase(databaseId); + it('works with a mixture of present and missing facets', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2573,13 +2468,10 @@ describe('DefaultEntitiesCatalog', () => { missing: [], }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'works when the entity is duplicated in search results, %p', - async databaseId => { - await createDatabase(databaseId); + it('works when the entity is duplicated in search results', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2621,28 +2513,22 @@ describe('DefaultEntitiesCatalog', () => { 'metadata.name': [{ value: 'one', count: 1 }], }, }); - }, - ); - - async function setupFacetsCatalog( - databaseId: TestDatabaseId, - entities: Entity[], - ) { - await createDatabase(databaseId); - for (const entity of entities) { - await addEntityToSearch(entity); - } - return new DefaultEntitiesCatalog({ - database: knex, - logger: mockServices.logger.mock(), - stitcher, }); - } - it.each(databases.eachSupportedId())( - 'excludes not-yet-stitched entities from filtered facets, %p', - async databaseId => { - await createDatabase(databaseId); + async function setupFacetsCatalog(entities: Entity[]) { + await createDatabase(); + for (const entity of entities) { + await addEntityToSearch(entity); + } + return new DefaultEntitiesCatalog({ + database: knex, + logger: mockServices.logger.mock(), + stitcher, + }); + } + + it('excludes not-yet-stitched entities from filtered facets', async () => { + await createDatabase(); await addEntityToSearch({ apiVersion: 'a', @@ -2701,13 +2587,10 @@ describe('DefaultEntitiesCatalog', () => { 'metadata.name': [{ value: 'stitched', count: 1 }], }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters with a predicate query, %p', - async databaseId => { - const catalog = await setupFacetsCatalog(databaseId, [ + it('filters with a predicate query', async () => { + const catalog = await setupFacetsCatalog([ { apiVersion: 'a', kind: 'Component', @@ -2742,13 +2625,10 @@ describe('DefaultEntitiesCatalog', () => { ]), }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters with a predicate query using $in, %p', - async databaseId => { - const catalog = await setupFacetsCatalog(databaseId, [ + it('filters with a predicate query using $in', async () => { + const catalog = await setupFacetsCatalog([ { apiVersion: 'a', kind: 'Component', @@ -2783,13 +2663,10 @@ describe('DefaultEntitiesCatalog', () => { ]), }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters with compound allOf filter, %p', - async databaseId => { - const catalog = await setupFacetsCatalog(databaseId, [ + it('filters with compound allOf filter', async () => { + const catalog = await setupFacetsCatalog([ { apiVersion: 'a', kind: 'Component', @@ -2826,13 +2703,10 @@ describe('DefaultEntitiesCatalog', () => { 'metadata.name': [{ value: 'one', count: 1 }], }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters with compound anyOf filter, %p', - async databaseId => { - const catalog = await setupFacetsCatalog(databaseId, [ + it('filters with compound anyOf filter', async () => { + const catalog = await setupFacetsCatalog([ { apiVersion: 'a', kind: 'Component', @@ -2872,13 +2746,10 @@ describe('DefaultEntitiesCatalog', () => { ]), }, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters with both filter and query combined, %p', - async databaseId => { - const catalog = await setupFacetsCatalog(databaseId, [ + it('filters with both filter and query combined', async () => { + const catalog = await setupFacetsCatalog([ { apiVersion: 'a', kind: 'Component', @@ -2911,10 +2782,10 @@ describe('DefaultEntitiesCatalog', () => { 'spec.type': [{ value: 'service', count: 1 }], }, }); - }, - ); - }); -}); + }); + }); + }, +); function entityFrom( name: string, diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index d659f8394e..c3244de20a 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -17,7 +17,6 @@ import { mockCredentials, mockServices, - TestDatabaseId, TestDatabases, } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; @@ -42,166 +41,162 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha'; jest.setTimeout(60_000); -describe('DefaultRefreshService', () => { - const defaultLogger = mockServices.logger.mock(); - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createDatabase( - databaseId: TestDatabaseId, - logger: LoggerService = defaultLogger, - ) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); - return { - knex, - processingDb: new DefaultProcessingDatabase({ - database: knex, - logger, - refreshInterval: () => 100, - events: mockServices.events.mock(), +describe.each(databases.eachSupportedId())( + 'DefaultRefreshService, %p', + databaseId => { + const defaultLogger = mockServices.logger.mock(); + + async function createDatabase(logger: LoggerService = defaultLogger) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return { + knex, + processingDb: new DefaultProcessingDatabase({ + database: knex, + logger, + refreshInterval: () => 100, + events: mockServices.events.mock(), + metrics: metricsServiceMock.mock(), + }), + catalogDb: new DefaultCatalogDatabase({ + database: knex, + logger, + }), + }; + } + + const createPopulatedEngine = async (options: { + db: ProcessingDatabase; + knex: Knex; + entities: Entity[]; + references: { [source: string]: string[] }; + entityProcessor?: (entity: Entity) => void; + }) => { + const { db, knex, entities, references, entityProcessor } = options; + + const entityMap = new Map( + entities.map(entity => [stringifyEntityRef(entity), entity]), + ); + + for (const entity of entities) { + await knex('refresh_state').insert({ + entity_id: uuid(), + entity_ref: stringifyEntityRef(entity), + unprocessed_entity: JSON.stringify(entity), + errors: '[]', + next_update_at: '2031-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + + const entitiesWithParent = new Set(Object.values(references).flat()); + for (const entityRef of entityMap.keys()) { + if (!entitiesWithParent.has(entityRef)) { + await knex( + 'refresh_state_references', + ).insert({ + source_key: 'ConfigLocationProvider', + target_entity_ref: entityRef, + }); + } + } + for (const [sourceRef, targetRefs] of Object.entries(references)) { + for (const targetRef of targetRefs) { + await knex( + 'refresh_state_references', + ).insert({ + source_entity_ref: sourceRef, + target_entity_ref: targetRef, + }); + } + } + + const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), { + knex, + logger: defaultLogger, metrics: metricsServiceMock.mock(), - }), - catalogDb: new DefaultCatalogDatabase({ - database: knex, - logger, - }), - }; - } - - const createPopulatedEngine = async (options: { - db: ProcessingDatabase; - knex: Knex; - entities: Entity[]; - references: { [source: string]: string[] }; - entityProcessor?: (entity: Entity) => void; - }) => { - const { db, knex, entities, references, entityProcessor } = options; - - const entityMap = new Map( - entities.map(entity => [stringifyEntityRef(entity), entity]), - ); - - for (const entity of entities) { - await knex('refresh_state').insert({ - entity_id: uuid(), - entity_ref: stringifyEntityRef(entity), - unprocessed_entity: JSON.stringify(entity), - errors: '[]', - next_update_at: '2031-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', }); - } + const engine = new DefaultCatalogProcessingEngine({ + config: new ConfigReader({}), + logger: defaultLogger, + processingDatabase: db, + knex: knex, + stitcher: stitcher, + scheduler: mockServices.scheduler(), + orchestrator: { + async process(request: EntityProcessingRequest) { + const entityRef = stringifyEntityRef(request.entity); + const entity = entityMap.get(entityRef); + if (!entity) { + throw new Error(`Unexpected entity: ${entityRef}`); + } + const deferredEntities = + references[entityRef]?.map(ref => { + const e = entityMap.get(ref); + if (!e) { + throw new Error(`Target entity not found: ${ref}`); + } + return { entity: e, locationKey: ref }; + }) || []; - const entitiesWithParent = new Set(Object.values(references).flat()); - for (const entityRef of entityMap.keys()) { - if (!entitiesWithParent.has(entityRef)) { - await knex( - 'refresh_state_references', - ).insert({ - source_key: 'ConfigLocationProvider', - target_entity_ref: entityRef, - }); - } - } - for (const [sourceRef, targetRefs] of Object.entries(references)) { - for (const targetRef of targetRefs) { - await knex( - 'refresh_state_references', - ).insert({ - source_entity_ref: sourceRef, - target_entity_ref: targetRef, - }); - } - } + entityProcessor?.(entity); - const stitcher = DefaultStitcher.fromConfig(new ConfigReader({}), { - knex, - logger: defaultLogger, - metrics: metricsServiceMock.mock(), - }); - const engine = new DefaultCatalogProcessingEngine({ - config: new ConfigReader({}), - logger: defaultLogger, - processingDatabase: db, - knex: knex, - stitcher: stitcher, - scheduler: mockServices.scheduler(), - orchestrator: { - async process(request: EntityProcessingRequest) { - const entityRef = stringifyEntityRef(request.entity); - const entity = entityMap.get(entityRef); - if (!entity) { - throw new Error(`Unexpected entity: ${entityRef}`); - } - const deferredEntities = - references[entityRef]?.map(ref => { - const e = entityMap.get(ref); - if (!e) { - throw new Error(`Target entity not found: ${ref}`); - } - return { entity: e, locationKey: ref }; - }) || []; - - entityProcessor?.(entity); - - return { - ok: true, - completedEntity: { - ...entity, - metadata: { - ...entity.metadata, - annotations: { - ...entity.metadata.annotations, - 'refresh-completed': 'true', + return { + ok: true, + completedEntity: { + ...entity, + metadata: { + ...entity.metadata, + annotations: { + ...entity.metadata.annotations, + 'refresh-completed': 'true', + }, }, }, - }, - relations: [], - errors: [], - deferredEntities, - state: {}, - refreshKeys: [], - }; + relations: [], + errors: [], + deferredEntities, + state: {}, + refreshKeys: [], + }; + }, }, - }, - createHash: () => createHash('sha1'), - pollingIntervalMs: 50, - events: mockServices.events.mock(), - metrics: metricsServiceMock.mock(), - }); + createHash: () => createHash('sha1'), + pollingIntervalMs: 50, + events: mockServices.events.mock(), + metrics: metricsServiceMock.mock(), + }); - return engine; - }; + return engine; + }; - const waitForRefresh = async (knex: Knex, entityRef: string) => { - for (;;) { - const [result] = await knex('refresh_state') - .where('entity_ref', entityRef) - .select(); + const waitForRefresh = async (knex: Knex, entityRef: string) => { + for (;;) { + const [result] = await knex('refresh_state') + .where('entity_ref', entityRef) + .select(); - const entity = result.processed_entity - ? (JSON.parse(result.processed_entity) as Entity) - : undefined; - if (entity?.metadata?.annotations?.['refresh-completed']) { - // Reset the annotation so that we can run another verification - delete entity.metadata.annotations['refresh-completed']; - await knex('refresh_state') - .update({ - processed_entity: JSON.stringify(entity), - }) - .where('entity_ref', entityRef); - return true; + const entity = result.processed_entity + ? (JSON.parse(result.processed_entity) as Entity) + : undefined; + if (entity?.metadata?.annotations?.['refresh-completed']) { + // Reset the annotation so that we can run another verification + delete entity.metadata.annotations['refresh-completed']; + await knex('refresh_state') + .update({ + processed_entity: JSON.stringify(entity), + }) + .where('entity_ref', entityRef); + return true; + } + await new Promise(resolve => setTimeout(resolve, 500)); } - await new Promise(resolve => setTimeout(resolve, 500)); - } - }; + }; - it.each(databases.eachSupportedId())( - 'should refresh the parent location, %p', - async databaseId => { - const { knex, processingDb, catalogDb } = await createDatabase( - databaseId, - ); + it('should refresh the parent location', async () => { + const { knex, processingDb, catalogDb } = await createDatabase(); const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ db: processingDb, @@ -239,15 +234,10 @@ describe('DefaultRefreshService', () => { ).resolves.toBe(true); await engine.stop(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should refresh the location further up the tree, %p', - async databaseId => { - const { knex, processingDb, catalogDb } = await createDatabase( - databaseId, - ); + it('should refresh the location further up the tree', async () => { + const { knex, processingDb, catalogDb } = await createDatabase(); const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ db: processingDb, @@ -293,16 +283,11 @@ describe('DefaultRefreshService', () => { ); await engine.stop(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should refresh even when parent has no changes', - async databaseId => { + it('should refresh even when parent has no changes', async () => { let secondRound = false; - const { knex, processingDb, catalogDb } = await createDatabase( - databaseId, - ); + const { knex, processingDb, catalogDb } = await createDatabase(); const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ db: processingDb, @@ -356,6 +341,6 @@ describe('DefaultRefreshService', () => { ).resolves.toBe(true); await engine.stop(); - }, - ); -}); + }); + }, +); diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 7092572ae6..ebd5daaf3f 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -19,7 +19,6 @@ import { wrapServer } from '@backstage/backend-openapi-utils/testUtils'; import { mockCredentials, mockServices, - TestDatabaseId, TestDatabases, } from '@backstage/backend-test-utils'; import type { Location } from '@backstage/catalog-client'; @@ -1680,59 +1679,59 @@ describe('createRouter readonly and raw json enabled', () => { }); }); -describe('POST /locations/by-query works end to end', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - async function createTestRouter(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); +describe.each(databases.eachSupportedId())( + 'POST /locations/by-query works end to end, %p', + databaseId => { + async function createTestRouter() { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); - const mockScmEvents = { - subscribe: jest.fn(), - publish: jest.fn(), - markEventActionTaken: jest.fn(), - }; + const mockScmEvents = { + subscribe: jest.fn(), + publish: jest.fn(), + markEventActionTaken: jest.fn(), + }; - const store = new DefaultLocationStore(knex, mockScmEvents, { - refresh: false, - unregister: false, - move: false, - }); - await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() }); + const store = new DefaultLocationStore(knex, mockScmEvents, { + refresh: false, + unregister: false, + move: false, + }); + await store.connect({ applyMutation: jest.fn(), refresh: jest.fn() }); - const locationService = new DefaultLocationService( - store, - { process: jest.fn() }, - { - allowedLocationTypes: ['url'], - defaultLocationConflictStrategy: 'reject', - }, - ); + const locationService = new DefaultLocationService( + store, + { process: jest.fn() }, + { + allowedLocationTypes: ['url'], + defaultLocationConflictStrategy: 'reject', + }, + ); - const router = await createRouter({ - locationService, - logger: mockServices.logger.mock(), - config: new ConfigReader(undefined), - auth: mockServices.auth(), - httpAuth: mockServices.httpAuth(), - orchestrator: { process: jest.fn() }, - permissionsService: mockServices.permissions(), - auditor: mockServices.auditor.mock(), - }); + const router = await createRouter({ + locationService, + logger: mockServices.logger.mock(), + config: new ConfigReader(undefined), + auth: mockServices.auth(), + httpAuth: mockServices.httpAuth(), + orchestrator: { process: jest.fn() }, + permissionsService: mockServices.permissions(), + auditor: mockServices.auditor.mock(), + }); - const errorMiddleware = MiddlewareFactory.create({ - logger: mockServices.logger.mock(), - config: mockServices.rootConfig(), - }); - router.use(errorMiddleware.error()); + const errorMiddleware = MiddlewareFactory.create({ + logger: mockServices.logger.mock(), + config: mockServices.rootConfig(), + }); + router.use(errorMiddleware.error()); - return { knex, app: express().use(router) }; - } + return { knex, app: express().use(router) }; + } - it.each(databases.eachSupportedId())( - 'paginates through locations correctly, %p', - async databaseId => { - const { knex, app } = await createTestRouter(databaseId); + it('paginates through locations correctly', async () => { + const { knex, app } = await createTestRouter(); // Insert 5 locations directly into the database const locations = [ @@ -1819,13 +1818,10 @@ describe('POST /locations/by-query works end to end', () => { totalItems: 5, pageInfo: {}, }); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'filters locations with query parameter, %p', - async databaseId => { - const { knex, app } = await createTestRouter(databaseId); + it('filters locations with query parameter', async () => { + const { knex, app } = await createTestRouter(); // Insert locations with different types const locations = [ @@ -1874,9 +1870,9 @@ describe('POST /locations/by-query works end to end', () => { totalItems: 2, pageInfo: {}, }); - }, - ); -}); + }); + }, +); function mockCursor(partialCursor?: Partial): Cursor { return { diff --git a/plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts b/plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts index 88acf68dd3..5ec9b39c71 100644 --- a/plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/DefaultStitcher.test.ts @@ -29,248 +29,246 @@ import { metricsServiceMock } from '@backstage/backend-test-utils/alpha'; jest.setTimeout(60_000); -describe('Stitcher', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); + +describe.each(databases.eachSupportedId())('Stitcher, %p', databaseId => { const logger = mockServices.logger.mock(); - it.each(databases.eachSupportedId())( - 'runs the happy path for %p', - async databaseId => { - const db = await databases.init(databaseId); - await applyDatabaseMigrations(db); + it('runs the happy path', async () => { + const db = await databases.init(databaseId); + await applyDatabaseMigrations(db); - const stitcher = new DefaultStitcher({ - knex: db, - logger, - strategy: { mode: 'immediate' }, - metrics: metricsServiceMock.mock(), - }); - let entities: DbFinalEntitiesRow[]; - let entity: Entity; + const stitcher = new DefaultStitcher({ + knex: db, + logger, + strategy: { mode: 'immediate' }, + metrics: metricsServiceMock.mock(), + }); + let entities: DbFinalEntitiesRow[]; + let entity: Entity; - await db('refresh_state').insert([ + await db('refresh_state').insert([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: db.fn.now(), + last_discovery_at: db.fn.now(), + }, + ]); + await db('refresh_state_references').insert([ + { source_key: 'a', target_entity_ref: 'k:ns/n' }, + ]); + await db('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/other', + }, + // handles and ignores duplicates + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/other', + }, + ]); + + await stitcher.stitch({ entityRefs: ['k:ns/n'] }); + + entities = await db('final_entities'); + + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entity).toEqual({ + relations: [ + { + type: 'looksAt', + targetRef: 'k:ns/other', + }, + ], + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at = entities[0].last_updated_at; + expect(last_updated_at).not.toBeNull(); + const firstHash = entities[0].hash; + + const search = await db('search'); + expect(search).toEqual( + expect.arrayContaining([ { entity_id: 'my-id', - entity_ref: 'k:ns/n', - unprocessed_entity: JSON.stringify({}), - processed_entity: JSON.stringify({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - }, - spec: { - k: 'v', - }, - }), - errors: '[]', - next_update_at: db.fn.now(), - last_discovery_at: db.fn.now(), + key: 'relations.looksat', + original_value: 'k:ns/other', + value: 'k:ns/other', }, - ]); - await db('refresh_state_references').insert([ - { source_key: 'a', target_entity_ref: 'k:ns/n' }, - ]); - await db('relations').insert([ { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/other', + entity_id: 'my-id', + key: 'apiversion', + original_value: 'a', + value: 'a', }, - // handles and ignores duplicates { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/other', + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', }, - ]); - - await stitcher.stitch({ entityRefs: ['k:ns/n'] }); - - entities = await db('final_entities'); - - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entity).toEqual({ - relations: [ - { - type: 'looksAt', - targetRef: 'k:ns/other', - }, - ], - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - uid: 'my-id', - }, - spec: { - k: 'v', - }, - }); - - expect(entity.metadata.etag).toEqual(entities[0].hash); - const last_updated_at = entities[0].last_updated_at; - expect(last_updated_at).not.toBeNull(); - const firstHash = entities[0].hash; - - const search = await db('search'); - expect(search).toEqual( - expect.arrayContaining([ - { - entity_id: 'my-id', - key: 'relations.looksat', - original_value: 'k:ns/other', - value: 'k:ns/other', - }, - { - entity_id: 'my-id', - key: 'apiversion', - original_value: 'a', - value: 'a', - }, - { - entity_id: 'my-id', - key: 'kind', - original_value: 'k', - value: 'k', - }, - { - entity_id: 'my-id', - key: 'metadata.name', - original_value: 'n', - value: 'n', - }, - { - entity_id: 'my-id', - key: 'metadata.namespace', - original_value: 'ns', - value: 'ns', - }, - { - entity_id: 'my-id', - key: 'metadata.uid', - original_value: 'my-id', - value: 'my-id', - }, - { - entity_id: 'my-id', - key: 'spec.k', - original_value: 'v', - value: 'v', - }, - ]), - ); - - // Re-stitch without any changes - await stitcher.stitch({ entityRefs: ['k:ns/n'] }); - - entities = await db('final_entities'); - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entities[0].hash).toEqual(firstHash); - expect(entity.metadata.etag).toEqual(firstHash); - - // Now add one more relation and re-stitch - await db('relations').insert([ { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', + entity_id: 'my-id', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, + { + entity_id: 'my-id', + key: 'metadata.namespace', + original_value: 'ns', + value: 'ns', + }, + { + entity_id: 'my-id', + key: 'metadata.uid', + original_value: 'my-id', + value: 'my-id', + }, + { + entity_id: 'my-id', + key: 'spec.k', + original_value: 'v', + value: 'v', + }, + ]), + ); + + // Re-stitch without any changes + await stitcher.stitch({ entityRefs: ['k:ns/n'] }); + + entities = await db('final_entities'); + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entities[0].hash).toEqual(firstHash); + expect(entity.metadata.etag).toEqual(firstHash); + + // Now add one more relation and re-stitch + await db('relations').insert([ + { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', + type: 'looksAt', + target_entity_ref: 'k:ns/third', + }, + ]); + + await stitcher.stitch({ entityRefs: ['k:ns/n'] }); + + entities = await db('final_entities'); + + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entity).toEqual({ + relations: expect.arrayContaining([ + { type: 'looksAt', - target_entity_ref: 'k:ns/third', + targetRef: 'k:ns/other', }, - ]); - - await stitcher.stitch({ entityRefs: ['k:ns/n'] }); - - entities = await db('final_entities'); - - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entity).toEqual({ - relations: expect.arrayContaining([ - { - type: 'looksAt', - targetRef: 'k:ns/other', - }, - { - type: 'looksAt', - targetRef: 'k:ns/third', - }, - ]), - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - uid: 'my-id', + { + type: 'looksAt', + targetRef: 'k:ns/third', }, - spec: { - k: 'v', + ]), + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entities[0].hash).not.toEqual(firstHash); + expect(entities[0].hash).toEqual(entity.metadata.etag); + + expect(await db('search')).toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + key: 'relations.looksat', + original_value: 'k:ns/other', + value: 'k:ns/other', }, - }); - - expect(entities[0].hash).not.toEqual(firstHash); - expect(entities[0].hash).toEqual(entity.metadata.etag); - - expect(await db('search')).toEqual( - expect.arrayContaining([ - { - entity_id: 'my-id', - key: 'relations.looksat', - original_value: 'k:ns/other', - value: 'k:ns/other', - }, - { - entity_id: 'my-id', - key: 'relations.looksat', - original_value: 'k:ns/third', - value: 'k:ns/third', - }, - { - entity_id: 'my-id', - key: 'apiversion', - original_value: 'a', - value: 'a', - }, - { - entity_id: 'my-id', - key: 'kind', - original_value: 'k', - value: 'k', - }, - { - entity_id: 'my-id', - key: 'metadata.name', - original_value: 'n', - value: 'n', - }, - { - entity_id: 'my-id', - key: 'metadata.namespace', - original_value: 'ns', - value: 'ns', - }, - { - entity_id: 'my-id', - key: 'metadata.uid', - original_value: 'my-id', - value: 'my-id', - }, - { - entity_id: 'my-id', - key: 'spec.k', - original_value: 'v', - value: 'v', - }, - ]), - ); - }, - ); + { + entity_id: 'my-id', + key: 'relations.looksat', + original_value: 'k:ns/third', + value: 'k:ns/third', + }, + { + entity_id: 'my-id', + key: 'apiversion', + original_value: 'a', + value: 'a', + }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, + { + entity_id: 'my-id', + key: 'metadata.name', + original_value: 'n', + value: 'n', + }, + { + entity_id: 'my-id', + key: 'metadata.namespace', + original_value: 'ns', + value: 'ns', + }, + { + entity_id: 'my-id', + key: 'metadata.uid', + original_value: 'my-id', + value: 'my-id', + }, + { + entity_id: 'my-id', + key: 'spec.k', + original_value: 'v', + value: 'v', + }, + ]), + ); + }); }); diff --git a/plugins/catalog-backend/src/tests/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts index 4b1153f3c6..8e95268acc 100644 --- a/plugins/catalog-backend/src/tests/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -42,717 +42,744 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - 'latest version correctly cascades deletions, %p', - async databaseId => { - const knex = await databases.init(databaseId); - await knex.migrate.latest({ directory: migrationsDir }); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('latest version correctly cascades deletions', async () => { + const knex = await databases.init(databaseId); + await knex.migrate.latest({ directory: migrationsDir }); - await knex - .insert({ - entity_id: 'i1', - entity_ref: 'k:ns/n1', - unprocessed_entity: '{}', - errors: '[]', - next_update_at: new Date(), - last_discovery_at: new Date(), - }) - .into('refresh_state'); - await knex - .insert({ - entity_id: 'i2', - entity_ref: 'k:ns/n2', - unprocessed_entity: '{}', - errors: '[]', - next_update_at: new Date(), - last_discovery_at: new Date(), - }) - .into('refresh_state'); - await knex - .insert({ - entity_id: 'i1', - hash: 'h', - final_entity: '{}', - entity_ref: 'k:ns/n1', - }) - .into('final_entities'); - await knex - .insert({ entity_id: 'i1', key: 'k1', value: 'v1' }) - .into('search'); - await knex - .insert({ - source_entity_ref: 'k:ns/n1', - target_entity_ref: 'k:ns/n2', - }) - .into('refresh_state_references'); - await knex - .insert({ - originating_entity_id: 'i1', - source_entity_ref: 'k:ns/n1', - target_entity_ref: 'k:ns/n2', - type: 't', - }) - .into('relations'); + await knex + .insert({ + entity_id: 'i1', + entity_ref: 'k:ns/n1', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: new Date(), + last_discovery_at: new Date(), + }) + .into('refresh_state'); + await knex + .insert({ + entity_id: 'i2', + entity_ref: 'k:ns/n2', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: new Date(), + last_discovery_at: new Date(), + }) + .into('refresh_state'); + await knex + .insert({ + entity_id: 'i1', + hash: 'h', + final_entity: '{}', + entity_ref: 'k:ns/n1', + }) + .into('final_entities'); + await knex + .insert({ entity_id: 'i1', key: 'k1', value: 'v1' }) + .into('search'); + await knex + .insert({ + source_entity_ref: 'k:ns/n1', + target_entity_ref: 'k:ns/n2', + }) + .into('refresh_state_references'); + await knex + .insert({ + originating_entity_id: 'i1', + source_entity_ref: 'k:ns/n1', + target_entity_ref: 'k:ns/n2', + type: 't', + }) + .into('relations'); - await knex.delete().from('refresh_state').where({ entity_id: 'i1' }); + await knex.delete().from('refresh_state').where({ entity_id: 'i1' }); - await expect(knex('search')).resolves.toEqual([]); - await expect(knex('refresh_state_references')).resolves.toEqual([]); - await expect(knex('relations')).resolves.toEqual([]); - await expect(knex('final_entities')).resolves.toEqual([]); - }, - ); + await expect(knex('search')).resolves.toEqual([]); + await expect(knex('refresh_state_references')).resolves.toEqual([]); + await expect(knex('relations')).resolves.toEqual([]); + await expect(knex('final_entities')).resolves.toEqual([]); + }); - it.each(databases.eachSupportedId())( - '20221109192547_search_add_original_value_column.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + it('20221109192547_search_add_original_value_column.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore( - knex, - '20221109192547_search_add_original_value_column.js', - ); + await migrateUntilBefore( + knex, + '20221109192547_search_add_original_value_column.js', + ); - await knex - .insert({ - entity_id: 'i', + await knex + .insert({ + entity_id: 'i', + entity_ref: 'k:ns/n', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: new Date(), + last_discovery_at: new Date(), + }) + .into('refresh_state'); + await knex + .insert({ entity_id: 'i', key: 'k1', value: 'v1' }) + .into('search'); + await knex + .insert({ entity_id: 'i', key: 'k2', value: null }) + .into('search'); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null }, + ]), + ); + + await migrateUpOnce(knex); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1', original_value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null, original_value: null }, + ]), + ); + + await migrateDownOnce(knex); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null }, + ]), + ); + + await knex.destroy(); + }); + + it('20221201085245_add_last_updated_at_in_final_entities.js', async () => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20221201085245_add_last_updated_at_in_final_entities.js', + ); + + await knex + .insert([ + { + entity_id: 'my-id', entity_ref: 'k:ns/n', - unprocessed_entity: '{}', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), errors: '[]', - next_update_at: new Date(), - last_discovery_at: new Date(), - }) - .into('refresh_state'); - await knex - .insert({ entity_id: 'i', key: 'k1', value: 'v1' }) - .into('search'); - await knex - .insert({ entity_id: 'i', key: 'k2', value: null }) - .into('search'); + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + ]) + .into('refresh_state'); - await expect(knex('search')).resolves.toEqual( - expect.arrayContaining([ - { entity_id: 'i', key: 'k1', value: 'v1' }, - { entity_id: 'i', key: 'k2', value: null }, - ]), - ); + await knex + .insert({ + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }) + .into('final_entities'); - await migrateUpOnce(knex); + await migrateUpOnce(knex); - await expect(knex('search')).resolves.toEqual( - expect.arrayContaining([ - { entity_id: 'i', key: 'k1', value: 'v1', original_value: 'v1' }, - { entity_id: 'i', key: 'k2', value: null, original_value: null }, - ]), - ); - - await migrateDownOnce(knex); - - await expect(knex('search')).resolves.toEqual( - expect.arrayContaining([ - { entity_id: 'i', key: 'k1', value: 'v1' }, - { entity_id: 'i', key: 'k2', value: null }, - ]), - ); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20221201085245_add_last_updated_at_in_final_entities.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await migrateUntilBefore( - knex, - '20221201085245_add_last_updated_at_in_final_entities.js', - ); - - await knex - .insert([ - { - entity_id: 'my-id', - entity_ref: 'k:ns/n', - unprocessed_entity: JSON.stringify({}), - processed_entity: JSON.stringify({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - }, - spec: { - k: 'v', - }, - }), - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }, - ]) - .into('refresh_state'); - - await knex - .insert({ + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', final_entity: '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', - }) - .into('final_entities'); + last_updated_at: null, + }, + ]), + ); - await migrateUpOnce(knex); + await migrateDownOnce(knex); - await expect(knex('final_entities')).resolves.toEqual( - expect.arrayContaining([ - { - entity_id: 'my-id', - hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', - stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', - final_entity: - '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', - last_updated_at: null, - }, - ]), - ); - - await migrateDownOnce(knex); - - await expect(knex('final_entities')).resolves.toEqual( - expect.arrayContaining([ - { - entity_id: 'my-id', - hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', - stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', - final_entity: - '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', - }, - ]), - ); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20230525141717_stitch_queue.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await migrateUntilBefore(knex, '20230525141717_stitch_queue.js'); - - await knex - .insert([ - { - entity_id: 'my-id', - entity_ref: 'k:ns/n', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }, - ]) - .into('refresh_state'); - await knex - .insert({ + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', - stitch_ticket: '', - final_entity: '{}', - }) - .into('final_entities'); + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }, + ]), + ); - await migrateUpOnce(knex); + await knex.destroy(); + }); - await expect(knex('refresh_state')).resolves.toEqual([ + it('20230525141717_stitch_queue.js', async () => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore(knex, '20230525141717_stitch_queue.js'); + + await knex + .insert([ { entity_id: 'my-id', entity_ref: 'k:ns/n', - location_key: null, unprocessed_entity: '{}', processed_entity: '{}', errors: '[]', - cache: null, - unprocessed_hash: null, - result_hash: null, - next_update_at: expect.anything(), - next_stitch_at: null, - next_stitch_ticket: null, - last_discovery_at: expect.anything(), + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), }, - ]); + ]) + .into('refresh_state'); + await knex + .insert({ + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '', + final_entity: '{}', + }) + .into('final_entities'); - await migrateDownOnce(knex); + await migrateUpOnce(knex); - await expect(knex('refresh_state')).resolves.toEqual([ + await expect(knex('refresh_state')).resolves.toEqual([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + location_key: null, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + cache: null, + unprocessed_hash: null, + result_hash: null, + next_update_at: expect.anything(), + next_stitch_at: null, + next_stitch_ticket: null, + last_discovery_at: expect.anything(), + }, + ]); + + await migrateDownOnce(knex); + + await expect(knex('refresh_state')).resolves.toEqual([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + location_key: null, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + cache: null, + unprocessed_hash: null, + result_hash: null, + next_update_at: expect.anything(), + last_discovery_at: expect.anything(), + }, + ]); + + await knex.destroy(); + }); + + it('20241003170511_alter_target_in_locations.js', async () => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20241003170511_alter_target_in_locations.js', + ); + + // Insert a simple URL before the migration + await knex + .insert({ + id: '1f99f7f6-1d87-4e8c-994a-8a2ba1d542f6', + type: 'url', + target: 'https://example.com', + }) + .into('locations'); + + await migrateUpOnce(knex); + + // Verify that the target column is now 'text' + const columnInfo = await knex('locations').columnInfo(); + expect(columnInfo.target.type).toBe('text'); + + // Insert a long URL (exceeding 255 characters) + await knex + .insert({ + id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', + type: 'url', + target: + 'https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-backstage-system-handles-it-making/test-this-alright-1/-/blob/main/catalog-info.yaml', + }) + .into('locations'); + + // Verify that both the simple and long URLs exist after the migration + await expect(knex('locations').where({ type: 'url' })).resolves.toEqual( + expect.arrayContaining([ { - entity_id: 'my-id', - entity_ref: 'k:ns/n', - location_key: null, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - cache: null, - unprocessed_hash: null, - result_hash: null, - next_update_at: expect.anything(), - last_discovery_at: expect.anything(), - }, - ]); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20241003170511_alter_target_in_locations.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await migrateUntilBefore( - knex, - '20241003170511_alter_target_in_locations.js', - ); - - // Insert a simple URL before the migration - await knex - .insert({ id: '1f99f7f6-1d87-4e8c-994a-8a2ba1d542f6', - type: 'url', target: 'https://example.com', - }) - .into('locations'); - - await migrateUpOnce(knex); - - // Verify that the target column is now 'text' - const columnInfo = await knex('locations').columnInfo(); - expect(columnInfo.target.type).toBe('text'); - - // Insert a long URL (exceeding 255 characters) - await knex - .insert({ - id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', type: 'url', + }, + { + id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', target: 'https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-backstage-system-handles-it-making/test-this-alright-1/-/blob/main/catalog-info.yaml', - }) - .into('locations'); + type: 'url', + }, + ]), + ); - // Verify that both the simple and long URLs exist after the migration - await expect(knex('locations').where({ type: 'url' })).resolves.toEqual( - expect.arrayContaining([ - { - id: '1f99f7f6-1d87-4e8c-994a-8a2ba1d542f6', - target: 'https://example.com', - type: 'url', - }, - { - id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', - target: - 'https://example.com/foo-bar-test-group/very-long-group-name-that-exceeds-255-characters-just-to-test-the-limits-of-url-length-in-the-catalog-info-yaml-file-and-see-how-the-backstage-system-handles-it-making/test-this-alright-1/-/blob/main/catalog-info.yaml', - type: 'url', - }, - ]), - ); + await expect(migrateDownOnce(knex)).rejects.toThrow( + `Migration aborted: Found 1 entries with 'target' exceeding 255 characters. Manual intervention required.`, + ); - await expect(migrateDownOnce(knex)).rejects.toThrow( - `Migration aborted: Found 1 entries with 'target' exceeding 255 characters. Manual intervention required.`, - ); + // Now remove the long URL + await knex('locations') + .where({ + id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', + }) + .del(); - // Now remove the long URL - await knex('locations') - .where({ - id: '1f99f6f7-1d87-4e8c-994a-2a8ba1d542f6', - }) - .del(); + // Retry the migration down after removing the long URL + await migrateDownOnce(knex); - // Retry the migration down after removing the long URL - await migrateDownOnce(knex); + // Verify that the column type has reverted to varchar + const revertedColumnInfo = await knex('locations').columnInfo(); + expect(revertedColumnInfo.target.type).toMatch( + /^(varchar|character varying)$/, + ); - // Verify that the column type has reverted to varchar - const revertedColumnInfo = await knex('locations').columnInfo(); - expect(revertedColumnInfo.target.type).toMatch( - /^(varchar|character varying)$/, - ); + // Verify that the short URL still exists in the table + await expect(knex('locations').where({ type: 'url' })).resolves.toEqual( + expect.arrayContaining([ + { + id: '1f99f7f6-1d87-4e8c-994a-8a2ba1d542f6', + target: 'https://example.com', + type: 'url', + }, + ]), + ); - // Verify that the short URL still exists in the table - await expect(knex('locations').where({ type: 'url' })).resolves.toEqual( - expect.arrayContaining([ - { - id: '1f99f7f6-1d87-4e8c-994a-8a2ba1d542f6', - target: 'https://example.com', - type: 'url', - }, - ]), - ); + await knex.destroy(); + }); - await knex.destroy(); - }, - ); + it('20241024104700_add_entity_ref_to_final_entities.js', async () => { + const knex = await databases.init(databaseId); - it.each(databases.eachSupportedId())( - '20241024104700_add_entity_ref_to_final_entities.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUntilBefore( + knex, + '20241024104700_add_entity_ref_to_final_entities.js', + ); - await migrateUntilBefore( - knex, - '20241024104700_add_entity_ref_to_final_entities.js', - ); + await knex + .insert({ + entity_id: 'id1', + entity_ref: 'k:ns/n', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }) + .into('refresh_state'); + await knex + .insert({ + entity_id: 'id2', + entity_ref: 'k:ns/n2', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }) + .into('refresh_state'); - await knex - .insert({ - entity_id: 'id1', - entity_ref: 'k:ns/n', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }) - .into('refresh_state'); - await knex - .insert({ - entity_id: 'id2', - entity_ref: 'k:ns/n2', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }) - .into('refresh_state'); + // Insert a simple entity before the migration + await knex + .insert({ + entity_id: 'id1', + hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: '{}', + }) + .into('final_entities'); - // Insert a simple entity before the migration - await knex - .insert({ + // verify that the entity_ref column is not present + const preColumnInfo = await knex('final_entities').columnInfo(); + expect(preColumnInfo.entity_ref).toBeUndefined(); + + await migrateUpOnce(knex); + + // verify that the entity_ref column has been added + const afterColumnInfo = await knex('final_entities').columnInfo(); + expect(afterColumnInfo.entity_ref).not.toBeUndefined(); + + // verify that the contents of the entity_ref column are correct + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'id1', hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', final_entity: '{}', + entity_ref: 'k:ns/n', + last_updated_at: null, + }, + ]), + ); + + // Verify that duplicates of entity_ref are not allowed. We lie about the + // ref to make it easier to trigger the problem. Also using a weak + // expectation due to sqlite flakiness; it rejects, but not necessarily + // with a valid error. + await expect( + knex + .insert({ + entity_id: 'id2', + entity_ref: 'k:ns/n', + hash: 'other', + stitch_ticket: 'other', + final_entity: '{}', }) - .into('final_entities'); + .into('final_entities'), + ).rejects.toEqual(expect.anything()); - // verify that the entity_ref column is not present - const preColumnInfo = await knex('final_entities').columnInfo(); - expect(preColumnInfo.entity_ref).toBeUndefined(); + await migrateDownOnce(knex); - await migrateUpOnce(knex); + // verify that the entity_ref column has been removed + const revertedColumnInfo = await knex('final_entities').columnInfo(); + expect(revertedColumnInfo.entity_ref).toBeUndefined(); - // verify that the entity_ref column has been added - const afterColumnInfo = await knex('final_entities').columnInfo(); - expect(afterColumnInfo.entity_ref).not.toBeUndefined(); + // verify that the contents are correct + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: 'id1', + hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: '{}', + last_updated_at: null, + }, + ]), + ); - // verify that the contents of the entity_ref column are correct - await expect(knex('final_entities')).resolves.toEqual( - expect.arrayContaining([ - { - entity_id: 'id1', - hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', - stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', - final_entity: '{}', - entity_ref: 'k:ns/n', - last_updated_at: null, - }, - ]), - ); + await knex.destroy(); + }); - // Verify that duplicates of entity_ref are not allowed. We lie about the - // ref to make it easier to trigger the problem. Also using a weak - // expectation due to sqlite flakiness; it rejects, but not necessarily - // with a valid error. - await expect( - knex - .insert({ - entity_id: 'id2', - entity_ref: 'k:ns/n', - hash: 'other', - stitch_ticket: 'other', - final_entity: '{}', - }) - .into('final_entities'), - ).rejects.toEqual(expect.anything()); + it('20241111000000_drop_redundant_indices.js', async () => { + const knex = await databases.init(databaseId); - await migrateDownOnce(knex); + await migrateUntilBefore(knex, '20241111000000_drop_redundant_indices.js'); - // verify that the entity_ref column has been removed - const revertedColumnInfo = await knex('final_entities').columnInfo(); - expect(revertedColumnInfo.entity_ref).toBeUndefined(); + await migrateUpOnce(knex); - // verify that the contents are correct - await expect(knex('final_entities')).resolves.toEqual( - expect.arrayContaining([ - { - entity_id: 'id1', - hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', - stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', - final_entity: '{}', - last_updated_at: null, - }, - ]), - ); + await migrateDownOnce(knex); - await knex.destroy(); - }, - ); + expect(true).toBe(true); + await knex.destroy(); + }); - it.each(databases.eachSupportedId())( - '20241111000000_drop_redundant_indices.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + it('20250401200503_update_refresh_state_columns.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore( - knex, - '20241111000000_drop_redundant_indices.js', - ); + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20250401200503_update_refresh_state_columns.js', + ); - await migrateUpOnce(knex); + // Insert a row with data into the refresh_state table + await knex('refresh_state').insert({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); - await migrateDownOnce(knex); - - expect(true).toBe(true); - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20250401200503_update_refresh_state_columns.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - // Run migrations up to just before the target migration - await migrateUntilBefore( - knex, - '20250401200503_update_refresh_state_columns.js', - ); - - // Insert a row with data into the refresh_state table - await knex('refresh_state').insert({ + // Verify the data before the migration + const preMigrationData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(preMigrationData).toEqual( + expect.objectContaining({ entity_id: 'test-id', entity_ref: 'k:ns/test', unprocessed_entity: JSON.stringify({ key: 'value' }), cache: JSON.stringify({ cacheKey: 'cacheValue' }), - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }); + }), + ); - // Verify the data before the migration - const preMigrationData = await knex('refresh_state') - .where({ entity_id: 'test-id' }) - .first(); - expect(preMigrationData).toEqual( - expect.objectContaining({ - entity_id: 'test-id', - entity_ref: 'k:ns/test', - unprocessed_entity: JSON.stringify({ key: 'value' }), - cache: JSON.stringify({ cacheKey: 'cacheValue' }), - }), - ); + // Run the migration + await migrateUpOnce(knex); - // Run the migration - await migrateUpOnce(knex); + // Verify the schema after the migration + const columnInfo = await knex('refresh_state').columnInfo(); + const expectedType = knex.client.config.client.includes('mysql') + ? 'longtext' + : 'text'; + expect(columnInfo.unprocessed_entity.type).toBe(expectedType); + expect(columnInfo.cache.type).toBe(expectedType); - // Verify the schema after the migration - const columnInfo = await knex('refresh_state').columnInfo(); - const expectedType = knex.client.config.client.includes('mysql') - ? 'longtext' - : 'text'; - expect(columnInfo.unprocessed_entity.type).toBe(expectedType); - expect(columnInfo.cache.type).toBe(expectedType); + // Verify the data after the migration + const postMigrationData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(postMigrationData).toEqual( + expect.objectContaining({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + }), + ); - // Verify the data after the migration - const postMigrationData = await knex('refresh_state') - .where({ entity_id: 'test-id' }) - .first(); - expect(postMigrationData).toEqual( - expect.objectContaining({ - entity_id: 'test-id', - entity_ref: 'k:ns/test', - unprocessed_entity: JSON.stringify({ key: 'value' }), - cache: JSON.stringify({ cacheKey: 'cacheValue' }), - }), - ); + // Roll back the migration + await migrateDownOnce(knex); - // Roll back the migration - await migrateDownOnce(knex); + // Verify the schema after rolling back + const revertedColumnInfo = await knex('refresh_state').columnInfo(); + expect(revertedColumnInfo.unprocessed_entity.type).toBe('text'); + expect(revertedColumnInfo.cache.type).toBe('text'); - // Verify the schema after rolling back - const revertedColumnInfo = await knex('refresh_state').columnInfo(); - expect(revertedColumnInfo.unprocessed_entity.type).toBe('text'); - expect(revertedColumnInfo.cache.type).toBe('text'); + // Verify the data after rolling back + const postRollbackData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(postRollbackData).toEqual( + expect.objectContaining({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + }), + ); - // Verify the data after rolling back - const postRollbackData = await knex('refresh_state') - .where({ entity_id: 'test-id' }) - .first(); - expect(postRollbackData).toEqual( - expect.objectContaining({ - entity_id: 'test-id', - entity_ref: 'k:ns/test', - unprocessed_entity: JSON.stringify({ key: 'value' }), - cache: JSON.stringify({ cacheKey: 'cacheValue' }), - }), - ); + await knex.destroy(); + }); - await knex.destroy(); - }, - ); + it('20250514000000_refresh_state_references_big_increments.js', async () => { + const knex = await databases.init(databaseId); - it.each(databases.eachSupportedId())( - '20250514000000_refresh_state_references_big_increments.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + const read = async () => { + return await knex('refresh_state_references') + .orderBy('id') + .then(rs => rs.map(r => ({ ...r, id: String(r.id) }))); + }; - const read = async () => { - return await knex('refresh_state_references') - .orderBy('id') - .then(rs => rs.map(r => ({ ...r, id: String(r.id) }))); - }; + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20250514000000_refresh_state_references_big_increments.js', + ); - // Run migrations up to just before the target migration - await migrateUntilBefore( - knex, - '20250514000000_refresh_state_references_big_increments.js', - ); + await knex('refresh_state').insert({ + entity_id: 'a', + entity_ref: 'k:ns/a', + unprocessed_entity: '{}', + cache: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); + await knex('refresh_state_references').insert({ + source_key: 'before', + target_entity_ref: 'k:ns/a', + }); - await knex('refresh_state').insert({ - entity_id: 'a', - entity_ref: 'k:ns/a', - unprocessed_entity: '{}', - cache: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }); - await knex('refresh_state_references').insert({ + await migrateUpOnce(knex); + + // can still insert with auto generated id in sequence + await knex('refresh_state_references').insert({ + source_key: 'after1', + target_entity_ref: 'k:ns/a', + }); + await expect(read()).resolves.toEqual([ + { + id: '1', source_key: 'before', + source_entity_ref: null, target_entity_ref: 'k:ns/a', - }); - - await migrateUpOnce(knex); - - // can still insert with auto generated id in sequence - await knex('refresh_state_references').insert({ + }, + { + id: '2', source_key: 'after1', + source_entity_ref: null, target_entity_ref: 'k:ns/a', - }); - await expect(read()).resolves.toEqual([ - { - id: '1', - source_key: 'before', - source_entity_ref: null, - target_entity_ref: 'k:ns/a', - }, - { - id: '2', - source_key: 'after1', - source_entity_ref: null, - target_entity_ref: 'k:ns/a', - }, - ]); + }, + ]); - await migrateDownOnce(knex); + await migrateDownOnce(knex); - // can still insert with auto generated id in sequence - await knex('refresh_state_references').insert({ + // can still insert with auto generated id in sequence + await knex('refresh_state_references').insert({ + source_key: 'after2', + target_entity_ref: 'k:ns/a', + }); + await expect(read()).resolves.toEqual([ + { + id: '1', + source_key: 'before', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + { + id: '2', + source_key: 'after1', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + { + id: '3', source_key: 'after2', + source_entity_ref: null, target_entity_ref: 'k:ns/a', - }); - await expect(read()).resolves.toEqual([ - { - id: '1', - source_key: 'before', - source_entity_ref: null, - target_entity_ref: 'k:ns/a', - }, - { - id: '2', - source_key: 'after1', - source_entity_ref: null, - target_entity_ref: 'k:ns/a', - }, - { - id: '3', - source_key: 'after2', - source_entity_ref: null, - target_entity_ref: 'k:ns/a', - }, - ]); + }, + ]); - await knex.destroy(); - }, - ); + await knex.destroy(); + }); - it.each(databases.eachSupportedId())( - '20260214000000_search_fk_final_entities.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + it('20260214000000_search_fk_final_entities.js', async () => { + const knex = await databases.init(databaseId); - // Run migrations up to just before the target migration - await migrateUntilBefore( - knex, - '20260214000000_search_fk_final_entities.js', - ); + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20260214000000_search_fk_final_entities.js', + ); - // Insert rows into refresh_state - await knex('refresh_state').insert([ - { - entity_id: 'id1', - entity_ref: 'component:default/service-a', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }, - { - entity_id: 'id2', - entity_ref: 'component:default/service-b', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }, - { - entity_id: 'id3', - entity_ref: 'api:default/my-api', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - }, - ]); + // Insert rows into refresh_state + await knex('refresh_state').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/service-a', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + { + entity_id: 'id2', + entity_ref: 'component:default/service-b', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + { + entity_id: 'id3', + entity_ref: 'api:default/my-api', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + ]); - // Insert rows into final_entities (only id1 and id2, not id3 - to test orphan deletion) - await knex('final_entities').insert([ - { - entity_id: 'id1', - entity_ref: 'component:default/service-a', - hash: 'hash1', - stitch_ticket: 'ticket1', - final_entity: '{}', - }, - { - entity_id: 'id2', - entity_ref: 'component:default/service-b', - hash: 'hash2', - stitch_ticket: 'ticket2', - final_entity: '{}', - }, - ]); + // Insert rows into final_entities (only id1 and id2, not id3 - to test orphan deletion) + await knex('final_entities').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/service-a', + hash: 'hash1', + stitch_ticket: 'ticket1', + final_entity: '{}', + }, + { + entity_id: 'id2', + entity_ref: 'component:default/service-b', + hash: 'hash2', + stitch_ticket: 'ticket2', + final_entity: '{}', + }, + ]); - // Insert rows into search table (with entity_id FK to refresh_state before migration) - await knex('search').insert([ + // Insert rows into search table (with entity_id FK to refresh_state before migration) + await knex('search').insert([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id1', + key: 'metadata.name', + value: 'service-a', + original_value: 'service-a', + }, + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + // id3 has no final_entities row, so this should be deleted during migration + { entity_id: 'id3', key: 'kind', value: 'api', original_value: 'API' }, + { + entity_id: 'id3', + key: 'metadata.name', + value: 'my-api', + original_value: 'my-api', + }, + // NULL entity_id row should survive the migration + { + entity_id: null, + key: 'global', + value: 'setting', + original_value: 'setting', + }, + ]); + + // Verify initial state + const preMigrationCount = await knex('search').count('* as count'); + expect(Number(preMigrationCount[0].count)).toBe(7); + + // Run the migration + await migrateUpOnce(knex); + + // Verify orphaned rows (id3) were deleted, but NULL entity_id row survived + const postMigrationRows = await knex('search'); + expect(postMigrationRows).toHaveLength(5); + expect(postMigrationRows).toEqual( + expect.arrayContaining([ + { + entity_id: null, + key: 'global', + value: 'setting', + original_value: 'setting', + }, { entity_id: 'id1', key: 'kind', @@ -777,559 +804,482 @@ describe('migrations', () => { value: 'service-b', original_value: 'service-b', }, - // id3 has no final_entities row, so this should be deleted during migration - { entity_id: 'id3', key: 'kind', value: 'api', original_value: 'API' }, - { - entity_id: 'id3', - key: 'metadata.name', - value: 'my-api', - original_value: 'my-api', - }, - // NULL entity_id row should survive the migration + ]), + ); + + // Verify FK is enforced: inserting with a nonexistent entity_id should be rejected + await expect( + knex('search').insert({ + entity_id: 'nonexistent', + key: 'kind', + value: 'test', + original_value: 'test', + }), + ).rejects.toEqual(expect.anything()); + + // Verify FK cascade works: deleting from final_entities should cascade to search + await knex('final_entities').where({ entity_id: 'id1' }).delete(); + const searchAfterDelete = await knex('search'); + expect(searchAfterDelete).toHaveLength(3); + expect(searchAfterDelete).toEqual( + expect.arrayContaining([ { entity_id: null, key: 'global', value: 'setting', original_value: 'setting', }, - ]); - - // Verify initial state - const preMigrationCount = await knex('search').count('* as count'); - expect(Number(preMigrationCount[0].count)).toBe(7); - - // Run the migration - await migrateUpOnce(knex); - - // Verify orphaned rows (id3) were deleted, but NULL entity_id row survived - const postMigrationRows = await knex('search'); - expect(postMigrationRows).toHaveLength(5); - expect(postMigrationRows).toEqual( - expect.arrayContaining([ - { - entity_id: null, - key: 'global', - value: 'setting', - original_value: 'setting', - }, - { - entity_id: 'id1', - key: 'kind', - value: 'component', - original_value: 'Component', - }, - { - entity_id: 'id1', - key: 'metadata.name', - value: 'service-a', - original_value: 'service-a', - }, - { - entity_id: 'id2', - key: 'kind', - value: 'component', - original_value: 'Component', - }, - { - entity_id: 'id2', - key: 'metadata.name', - value: 'service-b', - original_value: 'service-b', - }, - ]), - ); - - // Verify FK is enforced: inserting with a nonexistent entity_id should be rejected - await expect( - knex('search').insert({ - entity_id: 'nonexistent', - key: 'kind', - value: 'test', - original_value: 'test', - }), - ).rejects.toEqual(expect.anything()); - - // Verify FK cascade works: deleting from final_entities should cascade to search - await knex('final_entities').where({ entity_id: 'id1' }).delete(); - const searchAfterDelete = await knex('search'); - expect(searchAfterDelete).toHaveLength(3); - expect(searchAfterDelete).toEqual( - expect.arrayContaining([ - { - entity_id: null, - key: 'global', - value: 'setting', - original_value: 'setting', - }, - { - entity_id: 'id2', - key: 'kind', - value: 'component', - original_value: 'Component', - }, - { - entity_id: 'id2', - key: 'metadata.name', - value: 'service-b', - original_value: 'service-b', - }, - ]), - ); - - // Restore id1 for down migration test - await knex('final_entities').insert({ - entity_id: 'id1', - entity_ref: 'component:default/service-a', - hash: 'hash1', - stitch_ticket: 'ticket1', - final_entity: '{}', - }); - await knex('search').insert([ { - entity_id: 'id1', + entity_id: 'id2', key: 'kind', value: 'component', original_value: 'Component', }, - ]); + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + ]), + ); - // Delete id1 from refresh_state so it becomes an orphan for the down - // migration (exists in final_entities but not in refresh_state) - await knex('refresh_state').where({ entity_id: 'id1' }).delete(); + // Restore id1 for down migration test + await knex('final_entities').insert({ + entity_id: 'id1', + entity_ref: 'component:default/service-a', + hash: 'hash1', + stitch_ticket: 'ticket1', + final_entity: '{}', + }); + await knex('search').insert([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + ]); - // Run the down migration - await migrateDownOnce(knex); + // Delete id1 from refresh_state so it becomes an orphan for the down + // migration (exists in final_entities but not in refresh_state) + await knex('refresh_state').where({ entity_id: 'id1' }).delete(); - // Verify id1 search rows were cleaned up (orphan for refresh_state FK), - // while id2 and the NULL entity_id row survived - const revertedSearchRows = await knex('search'); - expect(revertedSearchRows).toHaveLength(3); - expect(revertedSearchRows).toEqual( - expect.arrayContaining([ - { - entity_id: null, - key: 'global', - value: 'setting', - original_value: 'setting', - }, - { - entity_id: 'id2', - key: 'kind', - value: 'component', - original_value: 'Component', - }, - { - entity_id: 'id2', - key: 'metadata.name', - value: 'service-b', - original_value: 'service-b', - }, - ]), - ); + // Run the down migration + await migrateDownOnce(knex); - // Verify FK is back to refresh_state: deleting from refresh_state should cascade - await knex('refresh_state').where({ entity_id: 'id2' }).delete(); - const afterRefreshStateDelete = await knex('search'); - expect(afterRefreshStateDelete).toEqual([ + // Verify id1 search rows were cleaned up (orphan for refresh_state FK), + // while id2 and the NULL entity_id row survived + const revertedSearchRows = await knex('search'); + expect(revertedSearchRows).toHaveLength(3); + expect(revertedSearchRows).toEqual( + expect.arrayContaining([ { entity_id: null, key: 'global', value: 'setting', original_value: 'setting', }, - ]); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20260215000000_move_stitch_queue.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - // Run migrations up to just before the target migration - await migrateUntilBefore(knex, '20260215000000_move_stitch_queue.js'); - - // Insert rows into refresh_state with stitch queue data - // Before migration, refresh_state has next_stitch_at and next_stitch_ticket columns - const stitchTime1 = new Date('2026-01-15T12:00:00.000Z'); - const stitchTime3 = new Date('2026-01-16T12:00:00.000Z'); - - await knex('refresh_state').insert([ { - entity_id: 'id1', - entity_ref: 'component:default/with-stitch', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - next_stitch_at: stitchTime1, - next_stitch_ticket: 'ticket-1', + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', }, { entity_id: 'id2', - entity_ref: 'component:default/no-stitch', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - next_stitch_at: null, - next_stitch_ticket: null, + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', }, - { - entity_id: 'id3', - entity_ref: 'component:default/orphan-stitch', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: knex.fn.now(), - last_discovery_at: knex.fn.now(), - next_stitch_at: stitchTime3, - next_stitch_ticket: 'ticket-3', - }, - ]); + ]), + ); - // Insert final_entities rows (with stitch_ticket column that will be dropped) - await knex('final_entities').insert([ - { - entity_id: 'id1', - entity_ref: 'component:default/with-stitch', - hash: 'h1', - stitch_ticket: 'old-ticket-1', - }, - { - entity_id: 'id2', - entity_ref: 'component:default/no-stitch', - hash: 'h2', - stitch_ticket: 'old-ticket-2', - }, - ]); + // Verify FK is back to refresh_state: deleting from refresh_state should cascade + await knex('refresh_state').where({ entity_id: 'id2' }).delete(); + const afterRefreshStateDelete = await knex('search'); + expect(afterRefreshStateDelete).toEqual([ + { + entity_id: null, + key: 'global', + value: 'setting', + original_value: 'setting', + }, + ]); - // Verify initial state - stitch_queue table should NOT exist yet - const preTableExists = await knex.schema.hasTable('stitch_queue'); - expect(preTableExists).toBe(false); + await knex.destroy(); + }); - // Run the migration - await migrateUpOnce(knex); + it('20260215000000_move_stitch_queue.js', async () => { + const knex = await databases.init(databaseId); - // Verify stitch_queue table was created - const postTableExists = await knex.schema.hasTable('stitch_queue'); - expect(postTableExists).toBe(true); + // Run migrations up to just before the target migration + await migrateUntilBefore(knex, '20260215000000_move_stitch_queue.js'); - // Verify next_stitch_at and next_stitch_ticket columns were removed from refresh_state - const refreshStateColumnInfo = await knex('refresh_state').columnInfo(); - expect(refreshStateColumnInfo.next_stitch_at).toBeUndefined(); - expect(refreshStateColumnInfo.next_stitch_ticket).toBeUndefined(); + // Insert rows into refresh_state with stitch queue data + // Before migration, refresh_state has next_stitch_at and next_stitch_ticket columns + const stitchTime1 = new Date('2026-01-15T12:00:00.000Z'); + const stitchTime3 = new Date('2026-01-16T12:00:00.000Z'); - // Verify stitch_ticket column was removed from final_entities - const finalEntitiesColumnInfo = await knex('final_entities').columnInfo(); - expect(finalEntitiesColumnInfo.stitch_ticket).toBeUndefined(); - - // Verify data was migrated correctly to stitch_queue - const stitchQueueAfterUp = await knex('stitch_queue') - .orderBy('entity_ref') - .select('entity_ref', 'stitch_ticket', 'next_stitch_at'); - - // Only entities with pending stitches should be in stitch_queue (id1 and id3) - expect(stitchQueueAfterUp).toEqual([ - { - entity_ref: 'component:default/orphan-stitch', - stitch_ticket: 'ticket-3', - next_stitch_at: expect.anything(), - }, - { - entity_ref: 'component:default/with-stitch', - stitch_ticket: 'ticket-1', - next_stitch_at: expect.anything(), - }, - ]); - - // Run the down migration - await migrateDownOnce(knex); - - // Verify stitch_queue table was dropped - const revertedTableExists = await knex.schema.hasTable('stitch_queue'); - expect(revertedTableExists).toBe(false); - - // Verify stitch_ticket column was restored to final_entities - const revertedFinalEntitiesColumnInfo = await knex( - 'final_entities', - ).columnInfo(); - expect(revertedFinalEntitiesColumnInfo.stitch_ticket).not.toBeUndefined(); - - // Verify next_stitch_at and next_stitch_ticket columns were restored to refresh_state - const revertedRefreshColumnInfo = await knex( - 'refresh_state', - ).columnInfo(); - expect(revertedRefreshColumnInfo.next_stitch_at).not.toBeUndefined(); - expect(revertedRefreshColumnInfo.next_stitch_ticket).not.toBeUndefined(); - - // Verify data was migrated back to refresh_state - const refreshStateAfterDown = await knex('refresh_state') - .orderBy('entity_id') - .select( - 'entity_id', - 'entity_ref', - 'next_stitch_at', - 'next_stitch_ticket', - ); - - // id1 should have stitch data restored - const id1RefreshRow = refreshStateAfterDown.find( - r => r.entity_id === 'id1', - ); - expect(id1RefreshRow?.next_stitch_at).not.toBeNull(); - expect(id1RefreshRow?.next_stitch_ticket).toBe('ticket-1'); - - // id2 should have no stitch data - const id2RefreshRow = refreshStateAfterDown.find( - r => r.entity_id === 'id2', - ); - expect(id2RefreshRow?.next_stitch_at).toBeNull(); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20260510000000_search_indices_and_dedup.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await migrateUntilBefore( - knex, - '20260510000000_search_indices_and_dedup.js', - ); - - // Set up FK dependencies: search → final_entities → refresh_state - await knex('refresh_state').insert({ - entity_id: 'e1', - entity_ref: 'k:ns/n1', + await knex('refresh_state').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/with-stitch', unprocessed_entity: '{}', + processed_entity: '{}', errors: '[]', next_update_at: knex.fn.now(), last_discovery_at: knex.fn.now(), - }); - await knex('final_entities').insert({ - entity_id: 'e1', - entity_ref: 'k:ns/n1', - hash: 'h1', - final_entity: '{}', - }); - - // Insert search rows with duplicates on (entity_id, key, value). - // Both copies of k1 share the same original_value so the surviving row - // is unambiguous across all database dedup strategies. - // The two null-value rows verify that null dedup does not conflate null - // with non-null or remove more rows than it should. - await knex('search').insert([ - { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'v1' }, // first copy - { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'v1' }, // duplicate — removed - { entity_id: 'e1', key: 'k2', value: null, original_value: null }, // null first — kept - { entity_id: 'e1', key: 'k2', value: null, original_value: null }, // null duplicate — removed - { entity_id: 'e1', key: 'k3', value: 'v3', original_value: 'v3' }, // unique — kept - ]); - - // Preconditions NOT met: dedup runs - await migrateUpOnce(knex); - - // 5 rows collapsed to 3 unique (entity_id, key, value) combinations - const rows = await knex('search').orderBy('key'); - expect(rows).toHaveLength(3); - expect(rows.map(r => r.key)).toEqual(['k1', 'k2', 'k3']); - expect(rows[0]).toEqual( - expect.objectContaining({ - key: 'k1', - value: 'v1', - original_value: 'v1', - }), - ); - // null-value row survives correctly - expect(rows[1]).toEqual( - expect.objectContaining({ - key: 'k2', - value: null, - original_value: null, - }), - ); - - // Unique constraint is now in place - await expect( - knex('search').insert({ - entity_id: 'e1', - key: 'k1', - value: 'v1', - original_value: 'extra', - }), - ).rejects.toEqual(expect.anything()); - - await migrateDownOnce(knex); - - // After rollback the unique constraint is gone — duplicates are allowed again - await expect( - knex('search').insert({ - entity_id: 'e1', - key: 'k1', - value: 'v1', - original_value: 'extra', - }), - ).resolves.not.toThrow(); - - await knex.destroy(); - }, - ); - - it.each(databases.eachSupportedId())( - '20260510000000_search_indices_and_dedup.js preconditions met (PG fast path), %p', - async databaseId => { - const knex = await databases.init(databaseId); - - // The fast path (skip dedup when unique index pre-exists) is a - // PostgreSQL-only code path that checks pg_index. - if (!knex.client.config.client.includes('pg')) { - await knex.destroy(); - return; - } - - await migrateUntilBefore( - knex, - '20260510000000_search_indices_and_dedup.js', - ); - - await knex('refresh_state').insert({ - entity_id: 'e1', - entity_ref: 'k:ns/n1', + next_stitch_at: stitchTime1, + next_stitch_ticket: 'ticket-1', + }, + { + entity_id: 'id2', + entity_ref: 'component:default/no-stitch', unprocessed_entity: '{}', + processed_entity: '{}', errors: '[]', next_update_at: knex.fn.now(), last_discovery_at: knex.fn.now(), - }); - await knex('final_entities').insert({ - entity_id: 'e1', - entity_ref: 'k:ns/n1', + next_stitch_at: null, + next_stitch_ticket: null, + }, + { + entity_id: 'id3', + entity_ref: 'component:default/orphan-stitch', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + next_stitch_at: stitchTime3, + next_stitch_ticket: 'ticket-3', + }, + ]); + + // Insert final_entities rows (with stitch_ticket column that will be dropped) + await knex('final_entities').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/with-stitch', hash: 'h1', - final_entity: '{}', - }); + stitch_ticket: 'old-ticket-1', + }, + { + entity_id: 'id2', + entity_ref: 'component:default/no-stitch', + hash: 'h2', + stitch_ticket: 'old-ticket-2', + }, + ]); - // Insert clean rows (no duplicates) so the unique index can be created - await knex('search').insert([ - { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'V1' }, - { entity_id: 'e1', key: 'k2', value: null, original_value: null }, - ]); + // Verify initial state - stitch_queue table should NOT exist yet + const preTableExists = await knex.schema.hasTable('stitch_queue'); + expect(preTableExists).toBe(false); - // Simulate a user who manually deduped and created the unique index - // before deploying this version - await knex.raw( - `CREATE UNIQUE INDEX search_entity_key_value_idx ON search (entity_id, key, value)`, + // Run the migration + await migrateUpOnce(knex); + + // Verify stitch_queue table was created + const postTableExists = await knex.schema.hasTable('stitch_queue'); + expect(postTableExists).toBe(true); + + // Verify next_stitch_at and next_stitch_ticket columns were removed from refresh_state + const refreshStateColumnInfo = await knex('refresh_state').columnInfo(); + expect(refreshStateColumnInfo.next_stitch_at).toBeUndefined(); + expect(refreshStateColumnInfo.next_stitch_ticket).toBeUndefined(); + + // Verify stitch_ticket column was removed from final_entities + const finalEntitiesColumnInfo = await knex('final_entities').columnInfo(); + expect(finalEntitiesColumnInfo.stitch_ticket).toBeUndefined(); + + // Verify data was migrated correctly to stitch_queue + const stitchQueueAfterUp = await knex('stitch_queue') + .orderBy('entity_ref') + .select('entity_ref', 'stitch_ticket', 'next_stitch_at'); + + // Only entities with pending stitches should be in stitch_queue (id1 and id3) + expect(stitchQueueAfterUp).toEqual([ + { + entity_ref: 'component:default/orphan-stitch', + stitch_ticket: 'ticket-3', + next_stitch_at: expect.anything(), + }, + { + entity_ref: 'component:default/with-stitch', + stitch_ticket: 'ticket-1', + next_stitch_at: expect.anything(), + }, + ]); + + // Run the down migration + await migrateDownOnce(knex); + + // Verify stitch_queue table was dropped + const revertedTableExists = await knex.schema.hasTable('stitch_queue'); + expect(revertedTableExists).toBe(false); + + // Verify stitch_ticket column was restored to final_entities + const revertedFinalEntitiesColumnInfo = await knex( + 'final_entities', + ).columnInfo(); + expect(revertedFinalEntitiesColumnInfo.stitch_ticket).not.toBeUndefined(); + + // Verify next_stitch_at and next_stitch_ticket columns were restored to refresh_state + const revertedRefreshColumnInfo = await knex('refresh_state').columnInfo(); + expect(revertedRefreshColumnInfo.next_stitch_at).not.toBeUndefined(); + expect(revertedRefreshColumnInfo.next_stitch_ticket).not.toBeUndefined(); + + // Verify data was migrated back to refresh_state + const refreshStateAfterDown = await knex('refresh_state') + .orderBy('entity_id') + .select( + 'entity_id', + 'entity_ref', + 'next_stitch_at', + 'next_stitch_ticket', ); - // Migration detects the existing unique index and skips the dedup step - await migrateUpOnce(knex); + // id1 should have stitch data restored + const id1RefreshRow = refreshStateAfterDown.find( + r => r.entity_id === 'id1', + ); + expect(id1RefreshRow?.next_stitch_at).not.toBeNull(); + expect(id1RefreshRow?.next_stitch_ticket).toBe('ticket-1'); - // Row count unchanged — no rows were removed by dedup - const rows = await knex('search').orderBy('key'); - expect(rows).toHaveLength(2); - expect(rows[0]).toEqual( - expect.objectContaining({ - key: 'k1', - value: 'v1', - original_value: 'V1', - }), - ); - expect(rows[1]).toEqual( - expect.objectContaining({ - key: 'k2', - value: null, - original_value: null, - }), - ); + // id2 should have no stitch data + const id2RefreshRow = refreshStateAfterDown.find( + r => r.entity_id === 'id2', + ); + expect(id2RefreshRow?.next_stitch_at).toBeNull(); - await migrateDownOnce(knex); + await knex.destroy(); + }); + + it('20260510000000_search_indices_and_dedup.js', async () => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20260510000000_search_indices_and_dedup.js', + ); + + // Set up FK dependencies: search → final_entities → refresh_state + await knex('refresh_state').insert({ + entity_id: 'e1', + entity_ref: 'k:ns/n1', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); + await knex('final_entities').insert({ + entity_id: 'e1', + entity_ref: 'k:ns/n1', + hash: 'h1', + final_entity: '{}', + }); + + // Insert search rows with duplicates on (entity_id, key, value). + // Both copies of k1 share the same original_value so the surviving row + // is unambiguous across all database dedup strategies. + // The two null-value rows verify that null dedup does not conflate null + // with non-null or remove more rows than it should. + await knex('search').insert([ + { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'v1' }, // first copy + { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'v1' }, // duplicate — removed + { entity_id: 'e1', key: 'k2', value: null, original_value: null }, // null first — kept + { entity_id: 'e1', key: 'k2', value: null, original_value: null }, // null duplicate — removed + { entity_id: 'e1', key: 'k3', value: 'v3', original_value: 'v3' }, // unique — kept + ]); + + // Preconditions NOT met: dedup runs + await migrateUpOnce(knex); + + // 5 rows collapsed to 3 unique (entity_id, key, value) combinations + const rows = await knex('search').orderBy('key'); + expect(rows).toHaveLength(3); + expect(rows.map(r => r.key)).toEqual(['k1', 'k2', 'k3']); + expect(rows[0]).toEqual( + expect.objectContaining({ + key: 'k1', + value: 'v1', + original_value: 'v1', + }), + ); + // null-value row survives correctly + expect(rows[1]).toEqual( + expect.objectContaining({ + key: 'k2', + value: null, + original_value: null, + }), + ); + + // Unique constraint is now in place + await expect( + knex('search').insert({ + entity_id: 'e1', + key: 'k1', + value: 'v1', + original_value: 'extra', + }), + ).rejects.toEqual(expect.anything()); + + await migrateDownOnce(knex); + + // After rollback the unique constraint is gone — duplicates are allowed again + await expect( + knex('search').insert({ + entity_id: 'e1', + key: 'k1', + value: 'v1', + original_value: 'extra', + }), + ).resolves.not.toThrow(); + + await knex.destroy(); + }); + + it('20260510000000_search_indices_and_dedup.js preconditions met (PG fast path)', async () => { + const knex = await databases.init(databaseId); + + // The fast path (skip dedup when unique index pre-exists) is a + // PostgreSQL-only code path that checks pg_index. + if (!knex.client.config.client.includes('pg')) { await knex.destroy(); - }, - ); + return; + } - it.each(databases.eachSupportedId())( - '20260403000000_add_location_entity_ref.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUntilBefore( + knex, + '20260510000000_search_indices_and_dedup.js', + ); - await migrateUntilBefore( - knex, - '20260403000000_add_location_entity_ref.js', - ); + await knex('refresh_state').insert({ + entity_id: 'e1', + entity_ref: 'k:ns/n1', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); + await knex('final_entities').insert({ + entity_id: 'e1', + entity_ref: 'k:ns/n1', + hash: 'h1', + final_entity: '{}', + }); - // The bootstrap location row was added by an earlier migration; after this - // migration it will have an empty-string placeholder for location_entity_ref. - const [bootstrapRow] = await knex('locations').where('type', 'bootstrap'); - expect(bootstrapRow).toBeDefined(); + // Insert clean rows (no duplicates) so the unique index can be created + await knex('search').insert([ + { entity_id: 'e1', key: 'k1', value: 'v1', original_value: 'V1' }, + { entity_id: 'e1', key: 'k2', value: null, original_value: null }, + ]); - // Insert a couple of non-bootstrap location rows to verify the backfill. - await knex('locations').insert([ - { - id: 'aaaaaaaa-0000-0000-0000-000000000001', - type: 'url', - target: 'https://example.com/a/catalog-info.yaml', - }, - { - id: 'aaaaaaaa-0000-0000-0000-000000000002', - type: 'url', - target: 'https://example.com/b/catalog-info.yaml', - }, - ]); + // Simulate a user who manually deduped and created the unique index + // before deploying this version + await knex.raw( + `CREATE UNIQUE INDEX search_entity_key_value_idx ON search (entity_id, key, value)`, + ); - // Verify the column does not yet exist - const columnsBefore = await knex('locations').columnInfo(); - expect(columnsBefore.location_entity_ref).toBeUndefined(); + // Migration detects the existing unique index and skips the dedup step + await migrateUpOnce(knex); - await migrateUpOnce(knex); + // Row count unchanged — no rows were removed by dedup + const rows = await knex('search').orderBy('key'); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual( + expect.objectContaining({ + key: 'k1', + value: 'v1', + original_value: 'V1', + }), + ); + expect(rows[1]).toEqual( + expect.objectContaining({ + key: 'k2', + value: null, + original_value: null, + }), + ); - // Column should now exist - const columnsAfter = await knex('locations').columnInfo(); - expect(columnsAfter.location_entity_ref).toBeDefined(); + await migrateDownOnce(knex); + await knex.destroy(); + }); - const rowsAfter = await knex('locations').orderBy('id').select(); + it('20260403000000_add_location_entity_ref.js', async () => { + const knex = await databases.init(databaseId); - // Helper matching the migration's own logic - function expectedRef(type: string, target: string): string { - return `location:default/generated-${createHash('sha1') - .update(`${type}:${target}`) - .digest('hex')}`.toLocaleLowerCase('en-US'); - } + await migrateUntilBefore(knex, '20260403000000_add_location_entity_ref.js'); - // Non-bootstrap rows get their entity ref backfilled - const rowA = rowsAfter.find( - r => r.id === 'aaaaaaaa-0000-0000-0000-000000000001', - ); - expect(rowA?.location_entity_ref).toBe( - expectedRef('url', 'https://example.com/a/catalog-info.yaml'), - ); + // The bootstrap location row was added by an earlier migration; after this + // migration it will have an empty-string placeholder for location_entity_ref. + const [bootstrapRow] = await knex('locations').where('type', 'bootstrap'); + expect(bootstrapRow).toBeDefined(); - const rowB = rowsAfter.find( - r => r.id === 'aaaaaaaa-0000-0000-0000-000000000002', - ); - expect(rowB?.location_entity_ref).toBe( - expectedRef('url', 'https://example.com/b/catalog-info.yaml'), - ); + // Insert a couple of non-bootstrap location rows to verify the backfill. + await knex('locations').insert([ + { + id: 'aaaaaaaa-0000-0000-0000-000000000001', + type: 'url', + target: 'https://example.com/a/catalog-info.yaml', + }, + { + id: 'aaaaaaaa-0000-0000-0000-000000000002', + type: 'url', + target: 'https://example.com/b/catalog-info.yaml', + }, + ]); - // The two targets produce distinct entity refs - expect(rowA?.location_entity_ref).not.toBe(rowB?.location_entity_ref); + // Verify the column does not yet exist + const columnsBefore = await knex('locations').columnInfo(); + expect(columnsBefore.location_entity_ref).toBeUndefined(); - // The bootstrap row gets an empty string placeholder (it will be removed - // in a future migration, so a real entity ref is not needed for it) - const bootstrapRowAfter = rowsAfter.find(r => r.type === 'bootstrap'); - expect(bootstrapRowAfter?.location_entity_ref).toBe(''); + await migrateUpOnce(knex); - // Rolling back removes the column - await migrateDownOnce(knex); + // Column should now exist + const columnsAfter = await knex('locations').columnInfo(); + expect(columnsAfter.location_entity_ref).toBeDefined(); - const columnsReverted = await knex('locations').columnInfo(); - expect(columnsReverted.location_entity_ref).toBeUndefined(); + const rowsAfter = await knex('locations').orderBy('id').select(); - await knex.destroy(); - }, - ); + // Helper matching the migration's own logic + function expectedRef(type: string, target: string): string { + return `location:default/generated-${createHash('sha1') + .update(`${type}:${target}`) + .digest('hex')}`.toLocaleLowerCase('en-US'); + } + + // Non-bootstrap rows get their entity ref backfilled + const rowA = rowsAfter.find( + r => r.id === 'aaaaaaaa-0000-0000-0000-000000000001', + ); + expect(rowA?.location_entity_ref).toBe( + expectedRef('url', 'https://example.com/a/catalog-info.yaml'), + ); + + const rowB = rowsAfter.find( + r => r.id === 'aaaaaaaa-0000-0000-0000-000000000002', + ); + expect(rowB?.location_entity_ref).toBe( + expectedRef('url', 'https://example.com/b/catalog-info.yaml'), + ); + + // The two targets produce distinct entity refs + expect(rowA?.location_entity_ref).not.toBe(rowB?.location_entity_ref); + + // The bootstrap row gets an empty string placeholder (it will be removed + // in a future migration, so a real entity ref is not needed for it) + const bootstrapRowAfter = rowsAfter.find(r => r.type === 'bootstrap'); + expect(bootstrapRowAfter?.location_entity_ref).toBe(''); + + // Rolling back removes the column + await migrateDownOnce(knex); + + const columnsReverted = await knex('locations').columnInfo(); + expect(columnsReverted.location_entity_ref).toBeUndefined(); + + await knex.destroy(); + }); }); diff --git a/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts b/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts index 6ac16f28f3..3987e5e937 100644 --- a/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts +++ b/plugins/catalog-backend/src/tests/performance/stitchingPerformance.test.ts @@ -144,14 +144,13 @@ class Tracker { } } -describePerformanceTest('stitchingPerformance', () => { - const databases = TestDatabases.create({ - ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], - }); +const databases = TestDatabases.create({ + ids: [/* 'MYSQL_8', */ 'POSTGRES_18', 'POSTGRES_14', 'SQLITE_3'], +}); - it.each(databases.eachSupportedId())( - 'runs stitching in immediate mode, %p', - async databaseId => { +describePerformanceTest('stitchingPerformance', () => { + describe.each(databases.eachSupportedId())('%p', databaseId => { + it('runs stitching in immediate mode', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -199,12 +198,9 @@ describePerformanceTest('stitchingPerformance', () => { await expect(tracker.completion()).resolves.toBeUndefined(); await backend.stop(); await knex.destroy(); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'runs stitching in deferred mode, %p', - async databaseId => { + it('runs stitching in deferred mode', async () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); @@ -252,6 +248,6 @@ describePerformanceTest('stitchingPerformance', () => { await expect(tracker.completion()).resolves.toBeUndefined(); await backend.stop(); await knex.destroy(); - }, - ); + }); + }); }); diff --git a/plugins/events-backend/src/migrations.test.ts b/plugins/events-backend/src/migrations.test.ts index 39be44947a..7e5d84901d 100644 --- a/plugins/events-backend/src/migrations.test.ts +++ b/plugins/events-backend/src/migrations.test.ts @@ -45,60 +45,54 @@ const databases = TestDatabases.create({ ids: ['POSTGRES_9', 'POSTGRES_14', 'POSTGRES_16'], }); -const maybeDescribe = - databases.eachSupportedId().length > 0 ? describe : describe.skip; +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20240523100528_init.js', async () => { + const knex = await databases.init(databaseId); -maybeDescribe('migrations', () => { - it.each(databases.eachSupportedId())( - '20240523100528_init.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); + await migrateUntilBefore(knex, '20240523100528_init.js'); + await migrateUpOnce(knex); - await migrateUntilBefore(knex, '20240523100528_init.js'); - await migrateUpOnce(knex); + await knex('event_bus_events').insert({ + topic: 'test', + created_by: 'abc', + data_json: JSON.stringify({ message: 'hello' }), + notified_subscribers: ['tester'], + }); + await knex('event_bus_subscriptions').insert({ + id: 'tester', + created_by: 'abc', + read_until: '5', + topics: ['test', 'test2'], + }); - await knex('event_bus_events').insert({ - topic: 'test', + await expect(knex('event_bus_events')).resolves.toEqual([ + { + id: '1', created_by: 'abc', + topic: 'test', data_json: JSON.stringify({ message: 'hello' }), + created_at: expect.anything(), notified_subscribers: ['tester'], - }); - await knex('event_bus_subscriptions').insert({ + }, + ]); + await expect(knex('event_bus_subscriptions')).resolves.toEqual([ + { id: 'tester', created_by: 'abc', + created_at: expect.anything(), + updated_at: expect.anything(), read_until: '5', topics: ['test', 'test2'], - }); + }, + ]); - await expect(knex('event_bus_events')).resolves.toEqual([ - { - id: '1', - created_by: 'abc', - topic: 'test', - data_json: JSON.stringify({ message: 'hello' }), - created_at: expect.anything(), - notified_subscribers: ['tester'], - }, - ]); - await expect(knex('event_bus_subscriptions')).resolves.toEqual([ - { - id: 'tester', - created_by: 'abc', - created_at: expect.anything(), - updated_at: expect.anything(), - read_until: '5', - topics: ['test', 'test2'], - }, - ]); + await migrateDownOnce(knex); - await migrateDownOnce(knex); + // This looks odd - you might expect a .toThrow at the end but that + // actually is flaky for some reason specifically on sqlite when + // performing multiple runs in sequence + await expect(knex('event_bus_events')).rejects.toEqual(expect.anything()); - // This looks odd - you might expect a .toThrow at the end but that - // actually is flaky for some reason specifically on sqlite when - // performing multiple runs in sequence - await expect(knex('event_bus_events')).rejects.toEqual(expect.anything()); - - await knex.destroy(); - }, - ); + await knex.destroy(); + }); }); diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 02205d8a65..d5203a47fc 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -22,7 +22,6 @@ import { } from '@backstage/backend-plugin-api'; import { TestBackend, - TestDatabaseId, TestDatabases, mockCredentials, mockServices, @@ -105,8 +104,15 @@ describe('eventsPlugin', () => { test: 'fake-ext', }); }); +}); - describe('event bus', () => { +const eventBusDatabases = TestDatabases.create({ + ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'], +}); + +describe.each(eventBusDatabases.eachSupportedId())( + 'event bus, %p', + databaseId => { class ReqHelper { private readonly backend: TestBackend; @@ -146,12 +152,8 @@ describe('eventsPlugin', () => { } } - const databases = TestDatabases.create({ - ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_14', 'POSTGRES_18'], - }); - - async function mockKnexFactory(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); + async function mockKnexFactory() { + const knex = await eventBusDatabases.init(databaseId); return mockServices.database.factory({ knex }); } @@ -163,278 +165,254 @@ describe('eventsPlugin', () => { } }); - it.each(databases.eachSupportedId())( - 'should be possible to publish events as a service, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); + it('should be possible to publish events as a service', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); - await helper - .publish('test', { n: 1 }) - .set('authorization', mockCredentials.none.header()) - .expect(401); + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.none.header()) + .expect(401); - await helper - .publish('test', { n: 1 }) - .set('authorization', mockCredentials.user.header()) - .expect(403); + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.user.header()) + .expect(403); - await helper - .publish('test', { n: 1 }) - .set('authorization', mockCredentials.service.header()) - .expect(204); // 204, since there are no subscribers - }, - ); + await helper + .publish('test', { n: 1 }) + .set('authorization', mockCredentials.service.header()) + .expect(204); // 204, since there are no subscribers + }); - it.each(databases.eachSupportedId())( - 'should be possible to subscribe as a service and receive an event, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); + it('should be possible to subscribe as a service and receive an event', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); - await helper - .subscribe('tester', ['test']) - .set('authorization', mockCredentials.none.header()) - .expect(401); + await helper + .subscribe('tester', ['test']) + .set('authorization', mockCredentials.none.header()) + .expect(401); - await helper - .subscribe('tester', ['test']) - .set('authorization', mockCredentials.user.header()) - .expect(403); + await helper + .subscribe('tester', ['test']) + .set('authorization', mockCredentials.user.header()) + .expect(403); - await helper.subscribe('tester', ['test']).expect(201); + await helper.subscribe('tester', ['test']).expect(201); - await helper - .readEvents('tester') - .set('authorization', mockCredentials.none.header()) - .expect(401); + await helper + .readEvents('tester') + .set('authorization', mockCredentials.none.header()) + .expect(401); - await helper - .readEvents('tester') - .set('authorization', mockCredentials.user.header()) - .expect(403); + await helper + .readEvents('tester') + .set('authorization', mockCredentials.user.header()) + .expect(403); - await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber + await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber - await helper.readEvents('tester').expect(200, { - events: [{ topic: 'test', payload: { n: 1 } }], - }); - }, - ); + await helper.readEvents('tester').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + }); - it.each(databases.eachSupportedId())( - 'should only send an event for each subscriber once, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); + it('should only send an event for each subscriber once', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); - // 2 subscribers - await helper.subscribe('tester-1', ['test']).expect(201); - await helper.subscribe('tester-2', ['test']).expect(201); + // 2 subscribers + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); - // A single event - await helper.publish('test', { n: 1 }).expect(201); + // A single event + await helper.publish('test', { n: 1 }).expect(201); - // Single client for subscriber 1 gets the event - await helper.readEvents('tester-1').expect(200, { - events: [{ topic: 'test', payload: { n: 1 } }], - }); + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); - // Two clients for subscriber 2, only one gets the event - const res1 = helper.readEvents('tester-2'); - const res2 = helper.readEvents('tester-2'); + // Two clients for subscriber 2, only one gets the event + const res1 = helper.readEvents('tester-2'); + const res2 = helper.readEvents('tester-2'); - const res = await Promise.race([res1, res2]); - expect(res.status).toBe(200); - expect(res.body).toEqual({ - events: [{ topic: 'test', payload: { n: 1 } }], - }); + const res = await Promise.race([res1, res2]); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + events: [{ topic: 'test', payload: { n: 1 } }], + }); - // Post another event, which triggers the other client to return - await helper.publish('test', { n: 2 }).expect(201); + // Post another event, which triggers the other client to return + await helper.publish('test', { n: 2 }).expect(201); - const otherRes = await Promise.all([res1, res2]).then(rs => - rs.find(r => r !== res), + const otherRes = await Promise.all([res1, res2]).then(rs => + rs.find(r => r !== res), + ); + expect(otherRes?.status).toBe(202); + + // Reading subscriber 2 should now return the second event only + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { n: 2 } }], + }); + }); + + it('should not notify subscribers that have already consumed the event', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); + + // A single event for each subscriber, that should not be sent to the other one + await helper + .publish( + 'test', + { for: 'tester-2' }, + { + notifiedSubscribers: ['tester-1'], + }, + ) + .expect(201); + await helper + .publish( + 'test', + { for: 'tester-1' }, + { + notifiedSubscribers: ['tester-2'], + }, + ) + .expect(201); + + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-1' } }], + }); + // Single client for subscriber 2 gets the event + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-2' } }], + }); + }); + + it('should return multiple events in order', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester', ['test']).expect(201); + + // A sequence of events published one at a time + for (let n = 0; n < 15; ++n) { + await helper.publish('test', { n }).expect(201); + } + + // Batch size it 10 + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 0 } }, + { topic: 'test', payload: { n: 1 } }, + { topic: 'test', payload: { n: 2 } }, + { topic: 'test', payload: { n: 3 } }, + { topic: 'test', payload: { n: 4 } }, + { topic: 'test', payload: { n: 5 } }, + { topic: 'test', payload: { n: 6 } }, + { topic: 'test', payload: { n: 7 } }, + { topic: 'test', payload: { n: 8 } }, + { topic: 'test', payload: { n: 9 } }, + ], + }); + + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 10 } }, + { topic: 'test', payload: { n: 11 } }, + { topic: 'test', payload: { n: 12 } }, + { topic: 'test', payload: { n: 13 } }, + { topic: 'test', payload: { n: 14 } }, + ], + }); + }); + + it('should skip publishing if all subscribers have already consumed the event', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); + + await helper.subscribe('tester', ['test']).expect(201); + + await helper + .publish( + 'test', + { for: 'tester-2' }, + { notifiedSubscribers: ['tester'] }, + ) + .expect(204); + }); + + it('should time out when no events are available', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); + await helper.subscribe('tester', ['test']).expect(201); + + jest.useFakeTimers({ + advanceTimers: true, + }); + + try { + // Can't use supertest for this one because it can't handle the partially blocking response + const res = await fetch( + `http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`, + { + headers: { + authorization: mockCredentials.service.header(), + }, + }, ); - expect(otherRes?.status).toBe(202); - // Reading subscriber 2 should now return the second event only - await helper.readEvents('tester-2').expect(200, { - events: [{ topic: 'test', payload: { n: 2 } }], - }); - }, - ); + expect(res.status).toBe(202); - it.each(databases.eachSupportedId())( - 'should not notify subscribers that have already consumed the event, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); + const reader = res.body!.getReader(); + const isClosed = () => + Promise.race([ + reader + .read() + .then(() => reader.closed) + .then(() => true), + new Promise(r => setTimeout(r, 100, false)), + ]); - // 2 subscribers - await helper.subscribe('tester-1', ['test']).expect(201); - await helper.subscribe('tester-2', ['test']).expect(201); + await expect(isClosed()).resolves.toBe(false); + await jest.advanceTimersByTimeAsync(30000); + await expect(isClosed()).resolves.toBe(false); + await jest.advanceTimersByTimeAsync(30000); + await expect(isClosed()).resolves.toBe(true); + } finally { + jest.useRealTimers(); + } + }); - // A single event for each subscriber, that should not be sent to the other one - await helper - .publish( - 'test', - { for: 'tester-2' }, - { - notifiedSubscribers: ['tester-1'], - }, - ) - .expect(201); - await helper - .publish( - 'test', - { for: 'tester-1' }, - { - notifiedSubscribers: ['tester-2'], - }, - ) - .expect(201); + it('should refuse listen without a subscription', async () => { + backend = await startTestBackend({ + features: [eventsPlugin, await mockKnexFactory()], + }); + const helper = new ReqHelper(backend); - // Single client for subscriber 1 gets the event - await helper.readEvents('tester-1').expect(200, { - events: [{ topic: 'test', payload: { for: 'tester-1' } }], - }); - // Single client for subscriber 2 gets the event - await helper.readEvents('tester-2').expect(200, { - events: [{ topic: 'test', payload: { for: 'tester-2' } }], - }); - }, - ); - - it.each(databases.eachSupportedId())( - 'should return multiple events in order, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); - - // 2 subscribers - await helper.subscribe('tester', ['test']).expect(201); - - // A sequence of events published one at a time - for (let n = 0; n < 15; ++n) { - await helper.publish('test', { n }).expect(201); - } - - // Batch size it 10 - await helper.readEvents('tester').expect(200, { - events: [ - { topic: 'test', payload: { n: 0 } }, - { topic: 'test', payload: { n: 1 } }, - { topic: 'test', payload: { n: 2 } }, - { topic: 'test', payload: { n: 3 } }, - { topic: 'test', payload: { n: 4 } }, - { topic: 'test', payload: { n: 5 } }, - { topic: 'test', payload: { n: 6 } }, - { topic: 'test', payload: { n: 7 } }, - { topic: 'test', payload: { n: 8 } }, - { topic: 'test', payload: { n: 9 } }, - ], - }); - - await helper.readEvents('tester').expect(200, { - events: [ - { topic: 'test', payload: { n: 10 } }, - { topic: 'test', payload: { n: 11 } }, - { topic: 'test', payload: { n: 12 } }, - { topic: 'test', payload: { n: 13 } }, - { topic: 'test', payload: { n: 14 } }, - ], - }); - }, - ); - - it.each(databases.eachSupportedId())( - 'should skip publishing if all subscribers have already consumed the event, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); - - await helper.subscribe('tester', ['test']).expect(201); - - await helper - .publish( - 'test', - { for: 'tester-2' }, - { notifiedSubscribers: ['tester'] }, - ) - .expect(204); - }, - ); - - it.each(databases.eachSupportedId())( - 'should time out when no events are available, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); - await helper.subscribe('tester', ['test']).expect(201); - - jest.useFakeTimers({ - advanceTimers: true, - }); - - try { - // Can't use supertest for this one because it can't handle the partially blocking response - const res = await fetch( - `http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`, - { - headers: { - authorization: mockCredentials.service.header(), - }, - }, - ); - - expect(res.status).toBe(202); - - const reader = res.body!.getReader(); - const isClosed = () => - Promise.race([ - reader - .read() - .then(() => reader.closed) - .then(() => true), - new Promise(r => setTimeout(r, 100, false)), - ]); - - await expect(isClosed()).resolves.toBe(false); - await jest.advanceTimersByTimeAsync(30000); - await expect(isClosed()).resolves.toBe(false); - await jest.advanceTimersByTimeAsync(30000); - await expect(isClosed()).resolves.toBe(true); - } finally { - jest.useRealTimers(); - } - }, - ); - - it.each(databases.eachSupportedId())( - 'should refuse listen without a subscription, %p', - async databaseId => { - backend = await startTestBackend({ - features: [eventsPlugin, await mockKnexFactory(databaseId)], - }); - const helper = new ReqHelper(backend); - - await helper.readEvents('nonexistent').expect(404); - }, - ); - }); -}); + await helper.readEvents('nonexistent').expect(404); + }); + }, +); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index 9e235fb152..1a7d297a8a 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -29,13 +29,10 @@ const databases = TestDatabases.create({ ids: ['POSTGRES_14', 'POSTGRES_18'], }); -const maybeDescribe = - databases.eachSupportedId().length > 0 ? describe : describe.skip; - -maybeDescribe('DatabaseEventBusStore', () => { - it.each(databases.eachSupportedId())( - 'should clean up old events, %p', - async databaseId => { +describe.each(databases.eachSupportedId())( + 'DatabaseEventBusStore, %p', + databaseId => { + it('should clean up old events', async () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, db }); @@ -79,12 +76,9 @@ maybeDescribe('DatabaseEventBusStore', () => { const { events: events3 } = await store.readSubscription('tester-3'); expect(events3.length).toBe(5); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should always clean up events outside the max age window, %p', - async databaseId => { + it('should always clean up events outside the max age window', async () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, @@ -132,12 +126,9 @@ maybeDescribe('DatabaseEventBusStore', () => { const { events: events3 } = await store.readSubscription('tester-3'); expect(events3.length).toBe(0); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should not clean up events within the min age window, %p', - async databaseId => { + it('should not clean up events within the min age window', async () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, @@ -162,12 +153,9 @@ maybeDescribe('DatabaseEventBusStore', () => { const { events: events1 } = await store.readSubscription('tester-1'); expect(events1.length).toBe(10); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should clean up a large number of events, %p', - async databaseId => { + it('should clean up a large number of events', async () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, @@ -197,12 +185,9 @@ maybeDescribe('DatabaseEventBusStore', () => { await expect(db('event_bus_events').count()).resolves.toEqual([ { count: '5' }, ]); - }, - ); + }); - it.each(databases.eachSupportedId())( - 'should perform well when looking up events by topic, %p', - async databaseId => { + it('should perform well when looking up events by topic', async () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, @@ -245,6 +230,6 @@ maybeDescribe('DatabaseEventBusStore', () => { ]); expect(duration).toBeLessThan(20); - }, - ); -}); + }); + }, +); diff --git a/plugins/notifications-backend/src/tests/migrations.test.ts b/plugins/notifications-backend/src/tests/migrations.test.ts index 65d8285e7f..5aaef28941 100644 --- a/plugins/notifications-backend/src/tests/migrations.test.ts +++ b/plugins/notifications-backend/src/tests/migrations.test.ts @@ -41,59 +41,56 @@ async function migrateUntilBefore(knex: Knex, target: string): Promise { jest.setTimeout(60_000); -describe('migrations', () => { - const databases = TestDatabases.create(); +const databases = TestDatabases.create(); - it.each(databases.eachSupportedId())( - '20221109192547_search_add_original_value_column.js, %p', - async databaseId => { - const knex = await databases.init(databaseId); +describe.each(databases.eachSupportedId())('migrations, %p', databaseId => { + it('20221109192547_search_add_original_value_column.js', async () => { + const knex = await databases.init(databaseId); - await migrateUntilBefore(knex, '20250317_addTopic.js'); + await migrateUntilBefore(knex, '20250317_addTopic.js'); - await knex - .insert({ + await knex + .insert({ + user: 'user1', + channel: 'channel1', + origin: 'origin1', + enabled: true, + }) + .into('user_settings'); + + await migrateUpOnce(knex); + + let rows = await knex('user_settings'); + let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled })); + + expect(normalized).toEqual( + expect.arrayContaining([ + expect.objectContaining({ user: 'user1', channel: 'channel1', origin: 'origin1', enabled: true, - }) - .into('user_settings'); + settings_key_hash: + '73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15', + topic: null, + }), + ]), + ); - await migrateUpOnce(knex); + await migrateDownOnce(knex); - let rows = await knex('user_settings'); - let normalized = rows.map(r => ({ ...r, enabled: !!r.enabled })); + rows = await knex('user_settings'); + normalized = rows.map(r => ({ ...r, enabled: !!r.enabled })); - expect(normalized).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - user: 'user1', - channel: 'channel1', - origin: 'origin1', - enabled: true, - settings_key_hash: - '73f97aff883b8b08a7f4e366234ef4f86827702b0016574ac4c1bf313c703d15', - topic: null, - }), - ]), - ); - - await migrateDownOnce(knex); - - rows = await knex('user_settings'); - normalized = rows.map(r => ({ ...r, enabled: !!r.enabled })); - - expect(normalized).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - user: 'user1', - channel: 'channel1', - origin: 'origin1', - enabled: true, - }), - ]), - ); - }, - ); + expect(normalized).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + user: 'user1', + channel: 'channel1', + origin: 'origin1', + enabled: true, + }), + ]), + ); + }); }); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index ac7c72cbfb..9f8cba802e 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - TestDatabaseId, - TestDatabases, - mockServices, -} from '@backstage/backend-test-utils'; +import { TestDatabases, mockServices } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { PgSearchHighlightOptions } from '../PgSearchEngine'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; @@ -38,701 +34,654 @@ const highlightOptions: PgSearchHighlightOptions = { jest.setTimeout(60_000); -describe('DatabaseDocumentStore', () => { - describe('unsupported', () => { - const databases = TestDatabases.create({ - ids: ['MYSQL_8', 'POSTGRES_9', 'SQLITE_3'], +const unsupportedDatabases = TestDatabases.create({ + ids: ['MYSQL_8', 'POSTGRES_9', 'SQLITE_3'], +}); + +describe.each(unsupportedDatabases.eachSupportedId())( + 'DatabaseDocumentStore unsupported, %p', + databaseId => { + it('should return support state', async () => { + const knex = await unsupportedDatabases.init(databaseId); + const supported = await DatabaseDocumentStore.supported(knex); + + expect(supported).toBe(false); }); - it.each(databases.eachSupportedId())( - 'should return support state, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const supported = await DatabaseDocumentStore.supported(knex); - - expect(supported).toBe(false); - }, - ); - - it.each(databases.eachSupportedId())( - 'should fail to create, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const databaseManager = mockServices.database({ knex }); - await expect( - async () => await DatabaseDocumentStore.create(databaseManager), - ).rejects.toThrow(); - }, - ); - }); - - describe('supported', () => { - const databases = TestDatabases.create({ - ids: ['POSTGRES_14'], + it('should fail to create', async () => { + const knex = await unsupportedDatabases.init(databaseId); + const databaseManager = mockServices.database({ knex }); + await expect( + async () => await DatabaseDocumentStore.create(databaseManager), + ).rejects.toThrow(); }); + }, +); - async function createStore(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); +const supportedDatabases = TestDatabases.create({ + ids: ['POSTGRES_14'], +}); + +describe.each(supportedDatabases.eachSupportedId())( + 'DatabaseDocumentStore supported, %p', + databaseId => { + async function createStore() { + const knex = await supportedDatabases.init(databaseId); const databaseManager = mockServices.database({ knex }); const store = await DatabaseDocumentStore.create(databaseManager); return { store, knex }; } - if (databases.eachSupportedId().length < 1) { - // Only execute tests if at least on database engine is available, e.g. if - // not in CI=1. it.each doesn't support an empty array. - return; - } + it('should return support state', async () => { + const knex = await supportedDatabases.init(databaseId); + const supported = await DatabaseDocumentStore.supported(knex); - it.each(databases.eachSupportedId())( - 'should return support state, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const supported = await DatabaseDocumentStore.supported(knex); + expect(supported).toBe(true); + }); - expect(supported).toBe(true); - }, - ); + it('should insert documents', async () => { + const { store, knex } = await createStore(); - it.each(databases.eachSupportedId())( - 'should insert documents, %p', - async databaseId => { - const { store, knex } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: 'TEXT 1', - location: 'LOCATION-1', - }, - { - title: 'TITLE 2', - text: 'TEXT 2', - location: 'LOCATION-2', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); - - expect( - await knex.count('*').where('type', 'my-type').from('documents'), - ).toEqual([{ count: '2' }]); - }, - ); - - it.each(databases.eachSupportedId())( - 'should insert truncated documents, %p', - async databaseId => { - const { store, knex } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: Array.from({ length: 100000 }) - .map(() => uuidv4()) // text tokens should be unique to overflow the tsvector indexing and trigger truncation - .join(' '), - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'my-type', true); - }); - - expect( - await knex.count('*').where('type', 'my-type').from('documents'), - ).toEqual([{ count: '1' }]); - }, - ); - - it.each(databases.eachSupportedId())( - 'should insert documents in batches, %p', - async databaseId => { - const { store, knex } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: 'TEXT 1', - location: 'LOCATION-1', - }, - { - title: 'TITLE 2', - text: 'TEXT 2', - location: 'LOCATION-2', - }, - ]); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 3', - text: 'TEXT 3', - location: 'LOCATION-3', - }, - { - title: 'TITLE 4', - text: 'TEXT 4', - location: 'LOCATION-4', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); - - expect( - await knex.count('*').where('type', 'my-type').from('documents'), - ).toEqual([{ count: '4' }]); - }, - ); - - it.each(databases.eachSupportedId())( - 'should clear index for type, %p', - async databaseId => { - const { store, knex } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'test', [ - { - title: 'TITLE 1', - text: 'TEXT 1', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'test'); - }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: 'TEXT 1', - location: 'LOCATION-1', - }, - { - title: 'TITLE 2', - text: 'TEXT 2', - location: 'LOCATION-2', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.completeInsert(tx, 'my-type'); - }); - - expect( - await knex.count('*').where('type', 'test').from('documents'), - ).toEqual([{ count: '1' }]); - expect( - await knex.count('*').where('type', 'my-type').from('documents'), - ).toEqual([{ count: '0' }]); - }, - ); - - it.each(databases.eachSupportedId())( - 'should return requested range, %p', - async databaseId => { - const { store } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'test', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - location: 'LOCATION-1', - }, - { - title: 'Hello World', - text: 'Around the world', - location: 'LOCATION-1', - }, - { - title: 'Another one', - text: 'From the next page', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'test'); - }); - - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - offset: 1, - limit: 1, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ { - total_count: '2', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - }, - rank: expect.any(Number), - type: 'test', + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', }, ]); - }, - ); + await store.completeInsert(tx, 'my-type'); + }); - it.each(databases.eachSupportedId())( - 'query by term, %p', - async databaseId => { - const { store } = await createStore(databaseId); + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '2' }]); + }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'test', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - location: 'LOCATION-1', - }, - { - title: 'Hello World', - text: 'Around the world', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'test'); - }); + it('should insert truncated documents', async () => { + const { store, knex } = await createStore(); - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); + await store.transaction(async tx => { + await store.prepareInsert(tx); - expect(rows).toEqual([ + await store.insertDocuments(tx, 'my-type', [ { - total_count: '2', - document: { - location: 'LOCATION-1', - text: 'Around the world', - title: 'Hello World', - }, - rank: expect.any(Number), - type: 'test', - }, - { - total_count: '2', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - }, - rank: expect.any(Number), - type: 'test', + title: 'TITLE 1', + text: Array.from({ length: 100000 }) + .map(() => uuidv4()) // text tokens should be unique to overflow the tsvector indexing and trigger truncation + .join(' '), + location: 'LOCATION-1', }, ]); - }, - ); + await store.completeInsert(tx, 'my-type', true); + }); - it.each(databases.eachSupportedId())( - 'query by term for specific type, %p', - async databaseId => { - const { store } = await createStore(databaseId); + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '1' }]); + }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'test', [ - { - title: 'Hello World', - text: 'Around the world', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'test'); - }); + it('should insert documents in batches', async () => { + const { store, knex } = await createStore(); - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - types: ['my-type'], - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ { - total_count: '1', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - }, - rank: expect.any(Number), - type: 'my-type', + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', }, ]); - }, - ); - - it.each(databases.eachSupportedId())( - 'query by term and filter by field, %p', - async databaseId => { - const { store } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Dolor sit amet', - text: 'Hello World', - myField: 'that', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Hello World', - text: 'Around the world', - location: 'LOCATION-1', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); - - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - fields: { myField: 'this' }, - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.insertDocuments(tx, 'my-type', [ { - total_count: '1', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - myField: 'this', - }, - rank: expect.any(Number), - type: 'my-type', + title: 'TITLE 3', + text: 'TEXT 3', + location: 'LOCATION-3', + }, + { + title: 'TITLE 4', + text: 'TEXT 4', + location: 'LOCATION-4', }, ]); - }, - ); + await store.completeInsert(tx, 'my-type'); + }); - it.each(databases.eachSupportedId())( - 'query by term and filter by field (any of), %p', - async databaseId => { - const { store } = await createStore(databaseId); + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '4' }]); + }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Dolor sit amet', - text: 'Hello World', - myField: 'that', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Hello World', - text: 'Around the world', - location: 'LOCATION-1', - }, - { - title: 'Sed ut perspiciatis', - text: 'Hello World', - myField: ['that', 'not'], - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Consectetur adipiscing', - text: 'Hello World', - myField: ['that', 'not', 'where'], - location: 'LOCATION-1', - } as unknown as IndexableDocument, - ]); - await store.completeInsert(tx, 'my-type'); - }); + it('should clear index for type', async () => { + const { store, knex } = await createStore(); - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - fields: { myField: ['this', 'that'] }, - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ { - total_count: '4', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - myField: 'this', - }, - rank: expect.any(Number), - type: 'my-type', - }, - { - total_count: '4', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Dolor sit amet', - myField: 'that', - }, - rank: expect.any(Number), - type: 'my-type', - }, - { - total_count: '4', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Sed ut perspiciatis', - myField: ['that', 'not'], - }, - rank: expect.any(Number), - type: 'my-type', - }, - { - total_count: '4', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Consectetur adipiscing', - myField: ['that', 'not', 'where'], - }, - rank: expect.any(Number), - type: 'my-type', + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', }, ]); - }, - ); - - it.each(databases.eachSupportedId())( - 'query by term and filter by fields, %p', - async databaseId => { - const { store } = await createStore(databaseId); - - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - myField: 'this', - otherField: 'another', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Dolor sit amet', - text: 'Hello World', - myField: 'this', - otherField: 'unknown', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - ]); - await store.completeInsert(tx, 'my-type'); - }); - - const rows = await store.transaction(tx => - store.query(tx, { - pgTerm: 'Hello & World', - fields: { myField: 'this', otherField: 'another' }, - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.completeInsert(tx, 'test'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ { - total_count: '1', - document: { - location: 'LOCATION-1', - text: 'Hello World', - title: 'Lorem Ipsum', - myField: 'this', - otherField: 'another', - }, - rank: expect.any(Number), - type: 'my-type', + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', }, ]); - }, - ); + await store.completeInsert(tx, 'my-type'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.completeInsert(tx, 'my-type'); + }); - it.each(databases.eachSupportedId())( - 'query without term and filter by field, %p', - async databaseId => { - const { store } = await createStore(databaseId); + expect( + await knex.count('*').where('type', 'test').from('documents'), + ).toEqual([{ count: '1' }]); + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '0' }]); + }); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'Lorem Ipsum', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - { - title: 'Dolor sit amet', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - ]); - await store.completeInsert(tx, 'my-type'); - }); + it('should return requested range', async () => { + const { store } = await createStore(); - const rows = await store.transaction(tx => - store.query(tx, { - fields: { myField: 'this' }, - offset: 0, - limit: 25, - normalization: 0, - options: highlightOptions, - }), - ); - - expect(rows).toEqual([ + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ { - total_count: '2', - document: { - title: 'Lorem Ipsum', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - rank: expect.any(Number), - type: 'my-type', + title: 'Lorem Ipsum', + text: 'Hello World', + location: 'LOCATION-1', }, { - total_count: '2', - document: { - title: 'Dolor sit amet', - text: 'Hello World', - myField: 'this', - location: 'LOCATION-1', - } as unknown as IndexableDocument, - rank: expect.any(Number), - type: 'my-type', + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + { + title: 'Another one', + text: 'From the next page', + location: 'LOCATION-1', }, ]); - }, - ); + await store.completeInsert(tx, 'test'); + }); - it.each(databases.eachSupportedId())( - 'should remove deleted documents and add new ones, %p', - async databaseId => { - const { store, knex } = await createStore(databaseId); + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + offset: 1, + limit: 1, + normalization: 0, + options: highlightOptions, + }), + ); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: 'TEXT 1', - location: 'LOCATION-1', - }, - { - title: 'TITLE 2', - text: 'TEXT 2', - location: 'LOCATION-2', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); + expect(rows).toEqual([ + { + total_count: '2', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + }, + rank: expect.any(Number), + type: 'test', + }, + ]); + }); - await expect( - knex.count('*').where('type', 'my-type').from('documents'), - ).resolves.toEqual([{ count: '2' }]); - const results_pre = await knex - .select('*') - .where('type', 'my-type') - .from('documents'); - expect(results_pre).toHaveLength(2); - expect(results_pre[0].document.title).toBe('TITLE 1'); - expect(results_pre[0].document.text).toBe('TEXT 1'); - expect(results_pre[1].document.title).toBe('TITLE 2'); + it('query by term', async () => { + const { store } = await createStore(); - await store.transaction(async tx => { - await store.prepareInsert(tx); - await store.insertDocuments(tx, 'my-type', [ - { - title: 'TITLE 1', - text: 'TEXT 1 updated', - location: 'LOCATION-1', - }, - { - title: 'TITLE 3', - text: 'TEXT 3', - location: 'LOCATION-3', - }, - ]); - await store.completeInsert(tx, 'my-type'); - }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + location: 'LOCATION-1', + }, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'test'); + }); - await expect( - knex.count('*').where('type', 'my-type').from('documents'), - ).resolves.toEqual([{ count: '2' }]); - const results_post = await knex - .select('*') - .where('type', 'my-type') - .from('documents'); - expect(results_post).toHaveLength(2); - expect(results_post[0].document.title).toBe('TITLE 1'); - expect(results_post[0].document.text).toBe('TEXT 1 updated'); - expect(results_post[1].document.title).toBe('TITLE 3'); - }, - ); - }); -}); + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '2', + document: { + location: 'LOCATION-1', + text: 'Around the world', + title: 'Hello World', + }, + rank: expect.any(Number), + type: 'test', + }, + { + total_count: '2', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + }, + rank: expect.any(Number), + type: 'test', + }, + ]); + }); + + it('query by term for specific type', async () => { + const { store } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'test'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + types: ['my-type'], + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '1', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }); + + it('query by term and filter by field', async () => { + const { store } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'that', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: 'this' }, + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '1', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }); + + it('query by term and filter by field (any of)', async () => { + const { store } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'that', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + { + title: 'Sed ut perspiciatis', + text: 'Hello World', + myField: ['that', 'not'], + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Consectetur adipiscing', + text: 'Hello World', + myField: ['that', 'not', 'where'], + location: 'LOCATION-1', + } as unknown as IndexableDocument, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: ['this', 'that'] }, + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '4', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + }, + rank: expect.any(Number), + type: 'my-type', + }, + { + total_count: '4', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Dolor sit amet', + myField: 'that', + }, + rank: expect.any(Number), + type: 'my-type', + }, + { + total_count: '4', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Sed ut perspiciatis', + myField: ['that', 'not'], + }, + rank: expect.any(Number), + type: 'my-type', + }, + { + total_count: '4', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Consectetur adipiscing', + myField: ['that', 'not', 'where'], + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }); + + it('query by term and filter by fields', async () => { + const { store } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + otherField: 'another', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + otherField: 'unknown', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: 'this', otherField: 'another' }, + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '1', + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + otherField: 'another', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }); + + it('query without term and filter by field', async () => { + const { store } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + { + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + fields: { myField: 'this' }, + offset: 0, + limit: 25, + normalization: 0, + options: highlightOptions, + }), + ); + + expect(rows).toEqual([ + { + total_count: '2', + document: { + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + rank: expect.any(Number), + type: 'my-type', + }, + { + total_count: '2', + document: { + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown as IndexableDocument, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }); + + it('should remove deleted documents and add new ones', async () => { + const { store, knex } = await createStore(); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + await expect( + knex.count('*').where('type', 'my-type').from('documents'), + ).resolves.toEqual([{ count: '2' }]); + const results_pre = await knex + .select('*') + .where('type', 'my-type') + .from('documents'); + expect(results_pre).toHaveLength(2); + expect(results_pre[0].document.title).toBe('TITLE 1'); + expect(results_pre[0].document.text).toBe('TEXT 1'); + expect(results_pre[1].document.title).toBe('TITLE 2'); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: 'TEXT 1 updated', + location: 'LOCATION-1', + }, + { + title: 'TITLE 3', + text: 'TEXT 3', + location: 'LOCATION-3', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + await expect( + knex.count('*').where('type', 'my-type').from('documents'), + ).resolves.toEqual([{ count: '2' }]); + const results_post = await knex + .select('*') + .where('type', 'my-type') + .from('documents'); + expect(results_post).toHaveLength(2); + expect(results_post[0].document.title).toBe('TITLE 1'); + expect(results_post[0].document.text).toBe('TEXT 1 updated'); + expect(results_post[1].document.title).toBe('TITLE 3'); + }); + }, +); diff --git a/plugins/search-backend-module-pg/src/database/util.test.ts b/plugins/search-backend-module-pg/src/database/util.test.ts index dd2bfde51a..8ce304ac44 100644 --- a/plugins/search-backend-module-pg/src/database/util.test.ts +++ b/plugins/search-backend-module-pg/src/database/util.test.ts @@ -18,45 +18,37 @@ import { queryPostgresMajorVersion } from './util'; jest.setTimeout(60_000); -describe('util', () => { - describe('unsupported', () => { - const databases = TestDatabases.create({ - ids: ['SQLITE_3', 'MYSQL_8'], - }); - - it.each(databases.eachSupportedId())( - 'should fail on get postgres major version, %p', - async databaseId => { - const knex = await databases.init(databaseId); - - await expect( - async () => await queryPostgresMajorVersion(knex), - ).rejects.toThrow(); - }, - ); - }); - - describe('supported', () => { - const databases = TestDatabases.create({ - ids: ['POSTGRES_18', 'POSTGRES_14'], - }); - - if (databases.eachSupportedId().length < 1) { - // Only execute tests if at least on database engine is available, e.g. if - // not in CI=1. it.each doesn't support an empty array. - return; - } - - it.each(databases.eachSupportedId())( - 'should get postgres major version, %p', - async databaseId => { - const knex = await databases.init(databaseId); - const expectedVersion = +databaseId.slice(9); - - await expect(queryPostgresMajorVersion(knex)).resolves.toBe( - expectedVersion, - ); - }, - ); - }); +const unsupportedDatabases = TestDatabases.create({ + ids: ['SQLITE_3', 'MYSQL_8'], }); + +describe.each(unsupportedDatabases.eachSupportedId())( + 'util unsupported, %p', + databaseId => { + it('should fail on get postgres major version', async () => { + const knex = await unsupportedDatabases.init(databaseId); + + await expect( + async () => await queryPostgresMajorVersion(knex), + ).rejects.toThrow(); + }); + }, +); + +const supportedDatabases = TestDatabases.create({ + ids: ['POSTGRES_18', 'POSTGRES_14'], +}); + +describe.each(supportedDatabases.eachSupportedId())( + 'util supported, %p', + databaseId => { + it('should get postgres major version', async () => { + const knex = await supportedDatabases.init(databaseId); + const expectedVersion = +databaseId.slice(9); + + await expect(queryPostgresMajorVersion(knex)).resolves.toBe( + expectedVersion, + ); + }); + }, +);