diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 1e795e9eb0..d2ac0871a9 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -576,6 +576,63 @@ describe('DefaultEntitiesCatalog', () => { }, ); + it.each(databases.eachSupportedId())( + 'handles inversion both for existing and missing keys, %p', + async databaseId => { + await createDatabase(databaseId); + + const entity1: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n1' }, + spec: { a: 'foo' }, + }; + const entity2: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n2' }, + spec: { a: 'bar', b: 'lonely' }, + }; + const entity3: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n3' }, + spec: { a: 'baz', b: 'only' }, + }; + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + await addEntityToSearch(entity3); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: mockServices.logger.mock(), + stitcher, + }); + + function f( + request: Omit, + ): Promise { + return catalog + .entities({ ...request, credentials: mockCredentials.none() }) + .then(response => + response.entities.map(e => e.metadata.name).toSorted(), + ); + } + + await expect( + f({ + filter: { key: 'spec.b', values: ['lonely'] }, + }), + ).resolves.toEqual(['n2']); + + await expect( + f({ + filter: { not: { key: 'spec.b', values: ['lonely'] } }, + }), + ).resolves.toEqual(['n1', 'n3']); + }, + ); + it.each(databases.eachSupportedId())( 'can order and combine with filtering, %p', async databaseId => { @@ -1825,6 +1882,21 @@ describe('DefaultEntitiesCatalog', () => { }, }); + await expect( + catalog.facets({ + facets: ['kind'], + filter: { key: 'metadata.name', values: ['two'] }, + credentials: mockCredentials.none(), + }), + ).resolves.toEqual({ + facets: { + kind: [ + { value: 'k', count: 1 }, + { value: 'k2', count: 1 }, + ], + }, + }); + await expect( catalog.facets({ facets: ['kind'], diff --git a/plugins/catalog-backend/src/tests/performance/getEntitiesPerformance.test.ts b/plugins/catalog-backend/src/tests/performance/getEntitiesPerformance.test.ts index 5721fd8218..5c853f8219 100644 --- a/plugins/catalog-backend/src/tests/performance/getEntitiesPerformance.test.ts +++ b/plugins/catalog-backend/src/tests/performance/getEntitiesPerformance.test.ts @@ -20,9 +20,14 @@ import { mockServices, startTestBackend, } from '@backstage/backend-test-utils'; -import { CatalogClient } from '@backstage/catalog-client'; +import { + CatalogApi, + CatalogClient, + GetEntitiesResponse, +} from '@backstage/catalog-client'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { Knex } from 'knex'; +import { default as catalogPlugin } from '../..'; import { applyDatabaseMigrations } from '../../database/migrations'; import { SyntheticLoadEntitiesProcessor, @@ -31,112 +36,189 @@ import { } from './lib/catalogModuleSyntheticLoadEntities'; import { describePerformanceTest, performanceTraceEnabled } from './lib/env'; +// #region Helpers + jest.setTimeout(600_000); +const databases = TestDatabases.create({ + ids: [/* 'MYSQL_8', */ 'POSTGRES_16', /* 'POSTGRES_12',*/ 'SQLITE_3'], + disableDocker: false, +}); + const traceLog: typeof console.log = performanceTraceEnabled ? console.log : () => {}; -const completionPolling = async (load: SyntheticLoadOptions, knex: Knex) => { - const { baseEntitiesCount, childrenCount } = load; - const expectedTotal = baseEntitiesCount + baseEntitiesCount * childrenCount; +async function createBackend( + knex: Knex, + load: SyntheticLoadOptions, +): Promise<{ + client: CatalogApi; + numberOfEntities: number; + stop: () => Promise; +}> { + await applyDatabaseMigrations(knex); - return new Promise((resolve, reject) => { - const interval = setInterval(async () => { - try { - const stitchedCount = await knex('final_entities') - .count({ count: '*' }) - .whereNotNull('final_entity') - .then(rows => Number(rows[0].count)); + const numberOfEntities = + load.baseEntitiesCount + load.baseEntitiesCount * load.childrenCount; - if (stitchedCount === expectedTotal) { - clearInterval(interval); - resolve(); - } - } catch (error) { - clearInterval(interval); - reject(error); - } - }, 1000); - }); -}; + traceLog(`Creating test backend with ${numberOfEntities} entities`); -describePerformanceTest('getEntitiesPerformanceTest', () => { - const databases = TestDatabases.create({ - ids: [/* 'MYSQL_8', */ 'POSTGRES_16', /* 'POSTGRES_12',*/ 'SQLITE_3'], - disableDocker: false, + const backend = await startTestBackend({ + features: [ + catalogPlugin, + mockServices.database.factory({ knex }), + createBackendModule({ + pluginId: 'catalog', + moduleId: 'synthetic-load-entities', + register(reg) { + reg.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + }, + async init({ catalog }) { + catalog.addEntityProvider( + new SyntheticLoadEntitiesProvider(load, {}), + ); + catalog.addProcessor(new SyntheticLoadEntitiesProcessor(load)); + }, + }); + }, + }), + ], }); - it.each(databases.eachSupportedId())( - 'fetch entities, %p', - async databaseId => { - const knex = await databases.init(databaseId); - await applyDatabaseMigrations(knex); + while ( + (await knex('final_entities') + .count({ count: '*' }) + .whereNotNull('final_entity') + .then(rows => Number(rows[0].count))) !== numberOfEntities + ) { + await new Promise(resolve => setTimeout(resolve, 200)); + } - const load: SyntheticLoadOptions = { + const client = new CatalogClient({ + discoveryApi: { + getBaseUrl: jest + .fn() + .mockResolvedValue( + `http://localhost:${backend.server.port()}/api/catalog`, + ), + }, + }); + + return { + client, + numberOfEntities, + stop: backend.stop.bind(backend), + }; +} + +// #endregion +// #region Tests + +describePerformanceTest('getEntities', () => { + let knex: Knex; + let backend: Awaited>; + + describe.each(databases.eachSupportedId())('burst reads, %p', databaseId => { + beforeAll(async () => { + knex = await databases.init(databaseId); + backend = await createBackend(knex, { baseEntitiesCount: 2000, baseRelationsCount: 3, baseRelationsSkew: 0.3, childrenCount: 3, - }; - - traceLog('Starting test backend'); - - const backend = await startTestBackend({ - features: [ - import('@backstage/plugin-catalog-backend/alpha'), - mockServices.database.factory({ knex }), - createBackendModule({ - pluginId: 'catalog', - moduleId: 'synthetic-load-entities', - register(reg) { - reg.registerInit({ - deps: { - catalog: catalogProcessingExtensionPoint, - }, - async init({ catalog }) { - catalog.addEntityProvider( - new SyntheticLoadEntitiesProvider(load, {}), - ); - catalog.addProcessor( - new SyntheticLoadEntitiesProcessor(load), - ); - }, - }); - }, - }), - ], }); + }); - const expectedTotal = - load.baseEntitiesCount + load.baseEntitiesCount * load.childrenCount; - traceLog(`Waiting for completion polling of ${expectedTotal} entities`); - - await expect(completionPolling(load, knex)).resolves.toBeUndefined(); - - const client = new CatalogClient({ - discoveryApi: { - getBaseUrl: jest - .fn() - .mockResolvedValue( - `http://localhost:${backend.server.port()}/api/catalog`, - ), - }, - }); - const start = Date.now(); - traceLog(`[${databaseId}] Starting to fetch ${expectedTotal} entities`); - - // Fetch all entities - const response = await client.getEntities(); - expect(response.items).toHaveLength(expectedTotal); - - const fetchDuration = Date.now() - start; - traceLog( - `[${databaseId}] Fetched ${expectedTotal} entities in ${fetchDuration}ms`, - ); - + afterAll(async () => { await backend.stop(); await knex.destroy(); - }, - ); + }); + + it('does a large burst read', async () => { + let response; + for (let i = 0; i < 10; ++i) { + response = await backend.client.getEntities(); + } + expect(response!.items).toHaveLength(backend.numberOfEntities); + }); + }); + + describe.each(databases.eachSupportedId())('filtering, %p', databaseId => { + beforeAll(async () => { + knex = await databases.init(databaseId); + backend = await createBackend(knex, { + baseEntitiesCount: 20000, + baseRelationsCount: 0, + baseRelationsSkew: 0, + childrenCount: 0, + }); + }); + + afterAll(async () => { + await backend.stop(); + await knex.destroy(); + }); + + it('single matching filter, all fields', async () => { + let response: GetEntitiesResponse; + for (let i = 0; i < 20; ++i) { + response = await backend.client.getEntities({ + filter: { kind: 'Location' }, + }); + } + expect(response!.items).toHaveLength(backend.numberOfEntities); + }); + + it('single matching filter, one field', async () => { + let response: GetEntitiesResponse; + for (let i = 0; i < 20; ++i) { + response = await backend.client.getEntities({ + filter: { kind: 'Location' }, + fields: ['kind'], + }); + } + expect(response!.items).toHaveLength(backend.numberOfEntities); + }); + + it('single non-matching filter', async () => { + let response: GetEntitiesResponse; + for (let i = 0; i < 20; ++i) { + response = await backend.client.getEntities({ + filter: { kind: 'NotALocation' }, + }); + } + expect(response!.items).toHaveLength(0); + }); + + it('complex filter, all fields', async () => { + let response: GetEntitiesResponse; + for (let i = 0; i < 20; ++i) { + response = await backend.client.getEntities({ + filter: [ + { + kind: 'Location', + }, + { + kind: 'Location', + 'metadata.name': new Array(100) + .fill(0) + .map((_, j) => `synthetic-${j}`), + }, + ...new Array(10).fill(0).map((_, j) => ({ + kind: 'NotALocation', + 'metadata.name': new Array(10) + .fill(0) + .map((__, k) => `no-match-${i}-${j}-${k}`), + })), + ], + }); + } + expect(response!.items).toHaveLength(backend.numberOfEntities); + }); + }); }); + +// #endregion