diff --git a/.changeset/improve-in-memory-catalog-client.md b/.changeset/improve-in-memory-catalog-client.md new file mode 100644 index 0000000000..9830b259ec --- /dev/null +++ b/.changeset/improve-in-memory-catalog-client.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +Improved the `InMemoryCatalogClient` test utility to support ordering, pagination, full-text search, and field projection for entity query methods. Also fixed `getEntityFacets` to correctly handle multi-valued fields. diff --git a/packages/catalog-client/package.json b/packages/catalog-client/package.json index fefca00189..5eb70b725b 100644 --- a/packages/catalog-client/package.json +++ b/packages/catalog-client/package.json @@ -51,11 +51,13 @@ "@backstage/catalog-model": "workspace:^", "@backstage/errors": "workspace:^", "cross-fetch": "^4.0.0", + "lodash": "^4.17.21", "uri-template": "^2.0.0" }, "devDependencies": { "@backstage/cli": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", + "@types/lodash": "^4.14.151", "msw": "^1.0.0" } } diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts index dbe3390747..d867ab1d28 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts @@ -25,6 +25,12 @@ const entity1: Entity = { namespace: 'default', name: 'e1', uid: 'u1', + annotations: { 'backstage.io/orphan': 'true' }, + tags: ['java', 'spring'], + }, + spec: { + type: 'service', + lifecycle: 'production', }, relations: [{ type: 'relatedTo', targetRef: 'secondcustomkind:default/e2' }], }; @@ -36,140 +42,889 @@ const entity2: Entity = { namespace: 'default', name: 'e2', uid: 'u2', + tags: ['python'], + }, + spec: { + type: 'library', + lifecycle: 'experimental', }, }; -const entities = [entity1, entity2]; +const entity3: Entity = { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'other', + name: 'e3', + uid: 'u3', + tags: ['java', 'quarkus'], + }, + spec: { + type: 'service', + lifecycle: 'production', + }, +}; + +const entity4: Entity = { + apiVersion: 'v1', + kind: 'SecondCustomKind', + metadata: { + namespace: 'default', + name: 'e4', + uid: 'u4', + }, + spec: { + type: 'website', + }, +}; + +const entities = [entity1, entity2, entity3, entity4]; describe('InMemoryCatalogClient', () => { - it('getEntities', async () => { - const client = new InMemoryCatalogClient({ entities }); + describe('getEntities', () => { + it('returns all entities without a filter', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect(client.getEntities()).resolves.toEqual({ items: entities }); + }); - await expect(client.getEntities()).resolves.toEqual({ items: entities }); + it('filters by metadata.name case-insensitively', async () => { + const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntities({ filter: { 'metadata.name': 'E1' } }), - ).resolves.toEqual({ items: [entity1] }); + await expect( + client.getEntities({ filter: { 'metadata.name': 'E1' } }), + ).resolves.toEqual({ items: [entity1] }); - await expect( - client.getEntities({ filter: { 'metadata.uid': 'u2' } }), - ).resolves.toEqual({ items: [entity2] }); + await expect( + client.getEntities({ filter: { 'metadata.uid': 'u2' } }), + ).resolves.toEqual({ items: [entity2] }); - await expect( - client.getEntities({ filter: { 'metadata.uid': 'U2' } }), - ).resolves.toEqual({ items: [entity2] }); + await expect( + client.getEntities({ filter: { 'metadata.uid': 'U2' } }), + ).resolves.toEqual({ items: [entity2] }); + }); - await expect( - client.getEntities({ - filter: { 'relations.relatedto': CATALOG_FILTER_EXISTS }, - }), - ).resolves.toEqual({ items: [entity1] }); + it('supports CATALOG_FILTER_EXISTS', async () => { + const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntities({ - filter: { 'relations.relatedTo': 'secondcustomkind:default/e2' }, - }), - ).resolves.toEqual({ items: [entity1] }); + await expect( + client.getEntities({ + filter: { 'relations.relatedto': CATALOG_FILTER_EXISTS }, + }), + ).resolves.toEqual({ items: [entity1] }); + }); - await expect( - client.getEntities({ - filter: { kind: ['not-existing', 'CustomKind'] }, - }), - ).resolves.toEqual({ items: [entity1] }); + it('supports filtering by relation value', async () => { + const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntities({ - filter: { kind: ['SecondCustomKind', 'also-not-existing'] }, - }), - ).resolves.toEqual({ items: [entity2] }); + await expect( + client.getEntities({ + filter: { 'relations.relatedTo': 'secondcustomkind:default/e2' }, + }), + ).resolves.toEqual({ items: [entity1] }); + }); + + it('supports array filter values as OR', async () => { + const client = new InMemoryCatalogClient({ entities }); + + await expect( + client.getEntities({ + filter: { kind: ['not-existing', 'CustomKind'] }, + }), + ).resolves.toEqual({ items: [entity1, entity3] }); + + await expect( + client.getEntities({ + filter: { kind: ['SecondCustomKind', 'also-not-existing'] }, + }), + ).resolves.toEqual({ items: [entity2, entity4] }); + }); + + it('supports multiple filter sets as OR', async () => { + const client = new InMemoryCatalogClient({ entities }); + + await expect( + client.getEntities({ + filter: [{ 'metadata.name': 'e1' }, { 'metadata.name': 'e3' }], + }), + ).resolves.toEqual({ items: [entity1, entity3] }); + }); + + it('orders results ascending', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'asc' }, + }); + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e1', + 'e2', + 'e3', + 'e4', + ]); + }); + + it('orders results descending', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'desc' }, + }); + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e4', + 'e3', + 'e2', + 'e1', + ]); + }); + + it('supports multi-level ordering', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: [ + { field: 'kind', order: 'asc' }, + { field: 'metadata.name', order: 'desc' }, + ], + }); + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e3', + 'e1', + 'e4', + 'e2', + ]); + }); + + it('sorts entities with missing order fields last regardless of direction', async () => { + const client = new InMemoryCatalogClient({ entities }); + + // entity4 has no lifecycle field - should be last in ascending + const asc = await client.getEntities({ + order: { field: 'spec.lifecycle', order: 'asc' }, + }); + expect(asc.items[asc.items.length - 1].metadata.name).toBe('e4'); + + // entity4 should also be last in descending + const desc = await client.getEntities({ + order: { field: 'spec.lifecycle', order: 'desc' }, + }); + expect(desc.items[desc.items.length - 1].metadata.name).toBe('e4'); + }); + + it('uses entity ref as stable tie-breaker when sort values are equal', async () => { + // entity1 (customkind:default/e1) and entity3 (customkind:other/e3) are both CustomKind + // entity2 (secondcustomkind:default/e2) and entity4 (secondcustomkind:default/e4) are both SecondCustomKind + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'kind', order: 'asc' }, + }); + // Within same kind, should be sorted by entity ref + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e1', + 'e3', + 'e2', + 'e4', + ]); + }); + + it('applies offset and limit', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'asc' }, + offset: 1, + limit: 2, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e2', 'e3']); + }); + + it('applies limit only', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'asc' }, + limit: 2, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e1', 'e2']); + }); + + it('applies offset only', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'asc' }, + offset: 2, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e3', 'e4']); + }); + + it('supports the after cursor for pagination', async () => { + const client = new InMemoryCatalogClient({ entities }); + + const cursor = Buffer.from( + JSON.stringify({ offset: 2, limit: 1 }), + ).toString('base64'); + + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'asc' }, + after: cursor, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e3']); + }); + + it('applies field projection', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + filter: { 'metadata.name': 'e1' }, + fields: ['kind', 'metadata.name'], + }); + expect(result.items).toEqual([ + { kind: 'CustomKind', metadata: { name: 'e1' } }, + ]); + }); + + it('applies field projection for nested paths', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + filter: { 'metadata.name': 'e1' }, + fields: ['metadata'], + }); + expect(result.items).toEqual([{ metadata: entity1.metadata }]); + }); + + it('handles dotted annotation keys in field projection', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + filter: { 'metadata.name': 'e1' }, + fields: ['metadata.annotations.backstage.io/orphan'], + }); + expect(result.items).toEqual([ + { + metadata: { + annotations: { 'backstage.io/orphan': 'true' }, + }, + }, + ]); + }); + + it('silently skips fields that do not exist on entities', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + filter: { 'metadata.name': 'e1' }, + fields: ['kind', 'spec.nonexistent', 'metadata.name'], + }); + expect(result.items).toEqual([ + { kind: 'CustomKind', metadata: { name: 'e1' } }, + ]); + }); + + it('applies field projection with ordering and pagination', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntities({ + order: { field: 'metadata.name', order: 'desc' }, + offset: 1, + limit: 2, + fields: ['metadata.name'], + }); + expect(result.items).toEqual([ + { metadata: { name: 'e3' } }, + { metadata: { name: 'e2' } }, + ]); + }); }); - it('getEntitiesByRefs', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntitiesByRefs({ + describe('getEntitiesByRefs', () => { + it('returns entities in the same order as refs', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntitiesByRefs({ + entityRefs: [ + 'secondcustomkind:default/e2', + 'customkind:missing/missing', + 'customkind:default/e1', + ], + }), + ).resolves.toEqual({ items: [entity2, undefined, entity1] }); + }); + + it('supports additional filter on refs', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntitiesByRefs({ + entityRefs: [ + 'secondcustomkind:default/e2', + 'customkind:missing/missing', + 'customkind:default/e1', + ], + filter: { 'metadata.uid': 'u1' }, + }), + ).resolves.toEqual({ items: [undefined, undefined, entity1] }); + }); + + it('applies field projection', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntitiesByRefs({ entityRefs: [ 'secondcustomkind:default/e2', 'customkind:missing/missing', 'customkind:default/e1', ], - }), - ).resolves.toEqual({ items: [entity2, undefined, entity1] }); - await expect( - client.getEntitiesByRefs({ - entityRefs: [ - 'secondcustomkind:default/e2', - 'customkind:missing/missing', - 'customkind:default/e1', + fields: ['kind', 'metadata.name'], + }); + expect(result.items).toEqual([ + { kind: 'SecondCustomKind', metadata: { name: 'e2' } }, + undefined, + { kind: 'CustomKind', metadata: { name: 'e1' } }, + ]); + }); + }); + + describe('queryEntities', () => { + it('returns all entities without parameters', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect(client.queryEntities()).resolves.toEqual({ + items: entities, + totalItems: 4, + pageInfo: {}, + }); + }); + + it('filters by entity filter', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.queryEntities({ filter: { 'metadata.uid': 'u2' } }), + ).resolves.toEqual({ + items: [entity2], + totalItems: 1, + pageInfo: {}, + }); + }); + + it('supports full-text search on the sort field by default', async () => { + const client = new InMemoryCatalogClient({ entities }); + // When no fullTextFilter.fields are given, the backend defaults to the + // sort field. Here we sort by spec.type and search for 'service'. + const result = await client.queryEntities({ + fullTextFilter: { term: 'service' }, + orderFields: { field: 'spec.type', order: 'asc' }, + }); + expect(result.items).toEqual([entity1, entity3]); + expect(result.totalItems).toBe(2); + }); + + it('defaults full-text search to metadata.uid when no sort field', async () => { + const client = new InMemoryCatalogClient({ entities }); + // Without orderFields, defaults to searching metadata.uid + const result = await client.queryEntities({ + fullTextFilter: { term: 'u1' }, + }); + expect(result.items).toEqual([entity1]); + }); + + it('supports full-text search limited to specific fields', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + fullTextFilter: { term: 'e1', fields: ['metadata.name'] }, + }); + expect(result.items).toEqual([entity1]); + }); + + it('supports full-text search across multiple explicit fields', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + fullTextFilter: { + term: 'e', + fields: ['metadata.name', 'metadata.uid'], + }, + orderFields: { field: 'metadata.name', order: 'asc' }, + }); + // All entities have 'e' in their name + expect(result.items).toEqual( + expect.arrayContaining([entity1, entity2, entity3, entity4]), + ); + }); + + it('supports case-insensitive full-text search', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + fullTextFilter: { term: 'SERVICE', fields: ['spec.type'] }, + }); + expect(result.items).toEqual([entity1, entity3]); + }); + + it('trims whitespace from full-text search term', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + fullTextFilter: { term: ' service ', fields: ['spec.type'] }, + }); + expect(result.items).toEqual([entity1, entity3]); + }); + + it('ignores empty full-text search term', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + fullTextFilter: { term: ' ' }, + }); + expect(result.items).toEqual(entities); + }); + + it('orders results by orderFields', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'desc' }, + }); + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e4', + 'e3', + 'e2', + 'e1', + ]); + }); + + it('supports multi-level ordering', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + orderFields: [ + { field: 'spec.type', order: 'asc' }, + { field: 'metadata.name', order: 'asc' }, ], - filter: { 'metadata.uid': 'u1' }, - }), - ).resolves.toEqual({ items: [undefined, undefined, entity1] }); - }); - - it('queryEntities', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect(client.queryEntities()).resolves.toEqual({ - items: entities, - totalItems: 2, - pageInfo: {}, + }); + expect(result.items.map(e => e.metadata.name)).toEqual([ + 'e2', + 'e1', + 'e3', + 'e4', + ]); }); - await expect( - client.queryEntities({ filter: { 'metadata.uid': 'u2' } }), - ).resolves.toEqual({ - items: [entity2], - totalItems: 1, - pageInfo: {}, + + it('uses entity ref as stable tie-breaker for equal sort values', async () => { + // entity1 and entity3 both have spec.type = 'service' + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + filter: { 'spec.type': 'service' }, + orderFields: { field: 'spec.type', order: 'asc' }, + }); + // customkind:default/e1 < customkind:other/e3 lexicographically + expect(result.items.map(e => e.metadata.name)).toEqual(['e1', 'e3']); + }); + + it('sorts entities with missing order fields last for both directions', async () => { + const client = new InMemoryCatalogClient({ entities }); + + const asc = await client.queryEntities({ + orderFields: { field: 'spec.lifecycle', order: 'asc' }, + }); + expect(asc.items[asc.items.length - 1].metadata.name).toBe('e4'); + + const desc = await client.queryEntities({ + orderFields: { field: 'spec.lifecycle', order: 'desc' }, + }); + expect(desc.items[desc.items.length - 1].metadata.name).toBe('e4'); + }); + + it('returns paginated results with limit', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + limit: 2, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e1', 'e2']); + expect(result.totalItems).toBe(4); + expect(result.pageInfo.nextCursor).toBeDefined(); + expect(result.pageInfo.prevCursor).toBeUndefined(); + }); + + it('returns paginated results with offset and limit', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + offset: 1, + limit: 2, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e2', 'e3']); + expect(result.totalItems).toBe(4); + expect(result.pageInfo.nextCursor).toBeDefined(); + expect(result.pageInfo.prevCursor).toBeDefined(); + }); + + it('paginates forward through all pages using cursors', async () => { + const client = new InMemoryCatalogClient({ entities }); + const allNames: string[] = []; + + const page1 = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + limit: 2, + }); + allNames.push(...page1.items.map(e => e.metadata.name)); + expect(page1.pageInfo.nextCursor).toBeDefined(); + expect(page1.pageInfo.prevCursor).toBeUndefined(); + + const page2 = await client.queryEntities({ + cursor: page1.pageInfo.nextCursor!, + limit: 2, + }); + allNames.push(...page2.items.map(e => e.metadata.name)); + expect(page2.pageInfo.nextCursor).toBeUndefined(); + expect(page2.pageInfo.prevCursor).toBeDefined(); + + expect(allNames).toEqual(['e1', 'e2', 'e3', 'e4']); + expect(page1.totalItems).toBe(4); + expect(page2.totalItems).toBe(4); + }); + + it('paginates backward using prevCursor', async () => { + const client = new InMemoryCatalogClient({ entities }); + + const page1 = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + offset: 2, + limit: 2, + }); + expect(page1.items.map(e => e.metadata.name)).toEqual(['e3', 'e4']); + expect(page1.pageInfo.prevCursor).toBeDefined(); + + const page0 = await client.queryEntities({ + cursor: page1.pageInfo.prevCursor!, + limit: 2, + }); + expect(page0.items.map(e => e.metadata.name)).toEqual(['e1', 'e2']); + expect(page0.pageInfo.prevCursor).toBeUndefined(); + }); + + it('handles page size of 1 for fine-grained pagination', async () => { + const client = new InMemoryCatalogClient({ entities }); + const allNames: string[] = []; + let cursor: string | undefined; + + const first = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + limit: 1, + }); + allNames.push(...first.items.map(e => e.metadata.name)); + cursor = first.pageInfo.nextCursor; + + while (cursor) { + const page = await client.queryEntities({ cursor, limit: 1 }); + allNames.push(...page.items.map(e => e.metadata.name)); + cursor = page.pageInfo.nextCursor; + } + + expect(allNames).toEqual(['e1', 'e2', 'e3', 'e4']); + }); + + it('produces stable pagination with clashing sort values', async () => { + // entity1 and entity3 both have spec.type = 'service' + // entity2 has spec.type = 'library', entity4 has spec.type = 'website' + const client = new InMemoryCatalogClient({ entities }); + const allNames: string[] = []; + let cursor: string | undefined; + + const first = await client.queryEntities({ + orderFields: { field: 'spec.type', order: 'asc' }, + limit: 1, + }); + allNames.push(...first.items.map(e => e.metadata.name)); + cursor = first.pageInfo.nextCursor; + + while (cursor) { + const page = await client.queryEntities({ cursor, limit: 1 }); + allNames.push(...page.items.map(e => e.metadata.name)); + cursor = page.pageInfo.nextCursor; + } + + // library < service < website, with tie-break by entity ref + expect(allNames).toEqual(['e2', 'e1', 'e3', 'e4']); + }); + + it('combines filter, full-text search, and ordering with pagination', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + filter: { kind: 'CustomKind' }, + fullTextFilter: { term: 'java', fields: ['metadata.tags'] }, + orderFields: { field: 'metadata.name', order: 'desc' }, + limit: 1, + }); + expect(result.items.map(e => e.metadata.name)).toEqual(['e3']); + expect(result.totalItems).toBe(2); + expect(result.pageInfo.nextCursor).toBeDefined(); + + const page2 = await client.queryEntities({ + cursor: result.pageInfo.nextCursor!, + limit: 1, + }); + expect(page2.items.map(e => e.metadata.name)).toEqual(['e1']); + expect(page2.pageInfo.nextCursor).toBeUndefined(); + }); + + it('applies field projection', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.queryEntities({ + filter: { 'metadata.name': 'e1' }, + fields: ['kind', 'metadata.name'], + }); + expect(result.items).toEqual([ + { kind: 'CustomKind', metadata: { name: 'e1' } }, + ]); + }); + + it('applies field projection with cursor pagination', async () => { + const client = new InMemoryCatalogClient({ entities }); + const page1 = await client.queryEntities({ + orderFields: { field: 'metadata.name', order: 'asc' }, + limit: 2, + fields: ['metadata.name'], + }); + expect(page1.items).toEqual([ + { metadata: { name: 'e1' } }, + { metadata: { name: 'e2' } }, + ]); + + const page2 = await client.queryEntities({ + cursor: page1.pageInfo.nextCursor!, + limit: 2, + fields: ['metadata.name'], + }); + expect(page2.items).toEqual([ + { metadata: { name: 'e3' } }, + { metadata: { name: 'e4' } }, + ]); + }); + + it('throws InputError for invalid cursor', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.queryEntities({ cursor: 'not-valid-base64!' }), + ).rejects.toThrow('Invalid cursor'); }); }); - it('streamEntities', async () => { - const client = new InMemoryCatalogClient({ entities }); - const stream = client.streamEntities(); - const results: Entity[][] = []; - for await (const page of stream) { - results.push(page); - } - expect(results).toEqual([entities]); - }); + describe('streamEntities', () => { + it('streams all entities', async () => { + const client = new InMemoryCatalogClient({ entities }); + const stream = client.streamEntities(); + const results: Entity[][] = []; + for await (const page of stream) { + results.push(page); + } + expect(results).toEqual([entities]); + }); - it('getEntityAncestors', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntityAncestors({ entityRef: 'secondcustomkind:default/e2' }), - ).resolves.toEqual({ - rootEntityRef: 'secondcustomkind:default/e2', - items: [{ entity: entity2, parentEntityRefs: [] }], + it('streams with filtering and ordering', async () => { + const client = new InMemoryCatalogClient({ entities }); + const stream = client.streamEntities({ + filter: { kind: 'CustomKind' }, + orderFields: { field: 'metadata.name', order: 'desc' }, + }); + const results: Entity[][] = []; + for await (const page of stream) { + results.push(page); + } + expect(results.flat().map(e => e.metadata.name)).toEqual(['e3', 'e1']); + }); + + it('streams in pages based on pageSize', async () => { + const client = new InMemoryCatalogClient({ entities }); + const stream = client.streamEntities({ pageSize: 2 }); + const results: Entity[][] = []; + for await (const page of stream) { + results.push(page); + } + expect(results).toHaveLength(2); + expect(results[0]).toHaveLength(2); + expect(results[1]).toHaveLength(2); + expect(results.flat()).toEqual(entities); + }); + + it('streams with full-text filter', async () => { + const client = new InMemoryCatalogClient({ entities }); + const stream = client.streamEntities({ + fullTextFilter: { term: 'library', fields: ['spec.type'] }, + }); + const results: Entity[][] = []; + for await (const page of stream) { + results.push(page); + } + expect(results.flat()).toEqual([entity2]); }); }); - it('getEntityByRef', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect( - client.getEntityByRef('secondcustomkind:default/e2'), - ).resolves.toEqual(entity2); - await expect( - client.getEntityByRef('customkind:missing/missing'), - ).resolves.toBeUndefined(); - }); - - it('removeEntityByUid', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect(client.getEntities()).resolves.toEqual({ - items: expect.arrayContaining([entity2]), + describe('getEntityAncestors', () => { + it('returns the entity with empty parent refs', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntityAncestors({ + entityRef: 'secondcustomkind:default/e2', + }), + ).resolves.toEqual({ + rootEntityRef: 'secondcustomkind:default/e2', + items: [{ entity: entity2, parentEntityRefs: [] }], + }); }); - await expect( - client.removeEntityByUid(entity2.metadata.uid!), - ).resolves.toBeUndefined(); - await expect(client.getEntities()).resolves.not.toEqual({ - items: expect.arrayContaining([entity2]), + + it('throws NotFoundError for missing entity', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntityAncestors({ entityRef: 'customkind:default/missing' }), + ).rejects.toThrow('not found'); }); }); - it('refreshEntity', async () => { - const client = new InMemoryCatalogClient({ entities }); - await expect( - client.refreshEntity('secondcustomkind:default/e2'), - ).resolves.toBeUndefined(); + describe('getEntityByRef', () => { + it('returns entity by ref string', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntityByRef('secondcustomkind:default/e2'), + ).resolves.toEqual(entity2); + }); + + it('returns undefined for missing entity', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.getEntityByRef('customkind:missing/missing'), + ).resolves.toBeUndefined(); + }); + }); + + describe('removeEntityByUid', () => { + it('removes an entity by uid', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect(client.getEntities()).resolves.toEqual({ + items: expect.arrayContaining([entity2]), + }); + await expect( + client.removeEntityByUid(entity2.metadata.uid!), + ).resolves.toBeUndefined(); + await expect(client.getEntities()).resolves.not.toEqual({ + items: expect.arrayContaining([entity2]), + }); + }); + + it('is a no-op for non-existent uid', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.removeEntityByUid('nonexistent'), + ).resolves.toBeUndefined(); + const result = await client.getEntities(); + expect(result.items).toHaveLength(4); + }); + }); + + describe('refreshEntity', () => { + it('is a no-op', async () => { + const client = new InMemoryCatalogClient({ entities }); + await expect( + client.refreshEntity('secondcustomkind:default/e2'), + ).resolves.toBeUndefined(); + }); + }); + + describe('getEntityFacets', () => { + it('returns facet counts for a simple field', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntityFacets({ facets: ['kind'] }); + expect(result.facets.kind).toEqual( + expect.arrayContaining([ + { value: 'CustomKind', count: 2 }, + { value: 'SecondCustomKind', count: 2 }, + ]), + ); + }); + + it('returns facet counts with filter', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntityFacets({ + facets: ['spec.type'], + filter: { kind: 'CustomKind' }, + }); + expect(result.facets['spec.type']).toEqual([ + { value: 'service', count: 2 }, + ]); + }); + + it('handles multi-valued fields correctly', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntityFacets({ + facets: ['metadata.tags'], + }); + // entity1 has ['java', 'spring'], entity2 has ['python'], entity3 has ['java', 'quarkus'] + // entity4 has no tags + expect(result.facets['metadata.tags']).toEqual( + expect.arrayContaining([ + { value: 'java', count: 2 }, + { value: 'spring', count: 1 }, + { value: 'python', count: 1 }, + { value: 'quarkus', count: 1 }, + ]), + ); + }); + + it('counts each distinct value once per entity', async () => { + // Entity with duplicate tag values in different forms - the search index + // might produce duplicates due to traverse behavior. Using a Set ensures + // each entity only contributes 1 to each facet value's count. + const dupEntity: Entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + namespace: 'default', + name: 'dup', + tags: ['java', 'java'], + }, + }; + const client = new InMemoryCatalogClient({ entities: [dupEntity] }); + const result = await client.getEntityFacets({ + facets: ['metadata.tags'], + }); + const javaFacet = result.facets['metadata.tags'].find( + f => f.value === 'java', + ); + // Should count as 1 per entity, not 2 for duplicate values + expect(javaFacet?.count).toBe(1); + }); + + it('returns multiple facets at once', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntityFacets({ + facets: ['kind', 'spec.type'], + }); + expect(Object.keys(result.facets)).toEqual(['kind', 'spec.type']); + expect(result.facets.kind).toHaveLength(2); + expect(result.facets['spec.type']).toEqual( + expect.arrayContaining([ + { value: 'service', count: 2 }, + { value: 'library', count: 1 }, + { value: 'website', count: 1 }, + ]), + ); + }); + + it('returns empty array for facets with no matching values', async () => { + const client = new InMemoryCatalogClient({ entities }); + const result = await client.getEntityFacets({ + facets: ['spec.nonexistent'], + }); + expect(result.facets['spec.nonexistent']).toEqual([]); + }); + }); + + describe('not implemented methods', () => { + it('throws NotImplementedError for location and validation methods', async () => { + const client = new InMemoryCatalogClient(); + await expect(client.getLocations()).rejects.toThrow('not implemented'); + await expect(client.getLocationById('id')).rejects.toThrow( + 'not implemented', + ); + await expect(client.getLocationByRef('ref')).rejects.toThrow( + 'not implemented', + ); + await expect(client.addLocation({ target: 'test' })).rejects.toThrow( + 'not implemented', + ); + await expect(client.removeLocationById('id')).rejects.toThrow( + 'not implemented', + ); + await expect(client.getLocationByEntity('ref')).rejects.toThrow( + 'not implemented', + ); + await expect(client.validateEntity(entity1, 'url:test')).rejects.toThrow( + 'not implemented', + ); + await expect( + client.analyzeLocation({ location: { type: 'url', target: 'test' } }), + ).rejects.toThrow('not implemented'); + }); }); }); diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts index a9db99ab77..a769f1996d 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts @@ -20,6 +20,7 @@ import { CATALOG_FILTER_EXISTS, CatalogApi, EntityFilterQuery, + EntityOrderQuery, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, GetEntitiesRequest, @@ -42,7 +43,12 @@ import { parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; -import { NotFoundError, NotImplementedError } from '@backstage/errors'; +import { + InputError, + NotFoundError, + NotImplementedError, +} from '@backstage/errors'; +import lodash from 'lodash'; // eslint-disable-next-line @backstage/no-relative-monorepo-imports import { traverse } from '../../../../plugins/catalog-backend/src/database/operations/stitcher/buildEntitySearch'; import type { @@ -51,6 +57,62 @@ import type { } from '@backstage/plugin-catalog-common'; import { DEFAULT_STREAM_ENTITIES_LIMIT } from '../constants.ts'; +function makeCursor(data: Record): string { + return btoa(JSON.stringify(data)); +} + +function parseCursor(cursor: string): Record { + return JSON.parse(atob(cursor)); +} + +// CATALOG_FILTER_EXISTS is a Symbol that doesn't survive JSON serialization, +// so we swap it with a sentinel string when encoding/decoding cursors. +const FILTER_EXISTS_SENTINEL = '\0CATALOG_FILTER_EXISTS'; + +function serializeFilterValue(v: unknown): unknown { + if (v === CATALOG_FILTER_EXISTS) return FILTER_EXISTS_SENTINEL; + if (Array.isArray(v)) { + return v.map(x => + x === CATALOG_FILTER_EXISTS ? FILTER_EXISTS_SENTINEL : x, + ); + } + return v; +} + +function deserializeFilterValue(v: unknown): unknown { + if (v === FILTER_EXISTS_SENTINEL) return CATALOG_FILTER_EXISTS; + if (Array.isArray(v)) { + return v.map(x => + x === FILTER_EXISTS_SENTINEL ? CATALOG_FILTER_EXISTS : x, + ); + } + return v; +} + +function serializeFilter(filter?: EntityFilterQuery): any[] | undefined { + if (!filter) return undefined; + return [filter] + .flat() + .map(f => + Object.fromEntries( + Object.entries(f).map(([k, v]) => [k, serializeFilterValue(v)]), + ), + ); +} + +function deserializeFilter(filter?: any[]): EntityFilterQuery | undefined { + if (!filter) return undefined; + return filter.map( + f => + Object.fromEntries( + Object.entries(f).map(([k, v]: [string, any]) => [ + k, + deserializeFilterValue(v), + ]), + ) as Record, + ); +} + function buildEntitySearch(entity: Entity) { const rows = traverse(entity); @@ -130,10 +192,120 @@ function createFilter( }; } +// Resolves a dot-separated field path against an entity, handling keys that +// themselves contain dots (e.g. annotation keys like "backstage.io/orphan"). +// This matches the backend's parseEntityTransformParams implementation. +function getPathArrayAndValue( + input: Entity, + field: string, +): [string[], unknown] { + return field.split('.').reduce( + ([pathArray, inputSubset], pathPart, index, fieldParts) => { + if (lodash.hasIn(inputSubset, pathPart)) { + return [pathArray.concat(pathPart), (inputSubset as any)[pathPart]]; + } else if (fieldParts[index + 1] !== undefined) { + fieldParts[index + 1] = `${pathPart}.${fieldParts[index + 1]}`; + return [pathArray, inputSubset]; + } + return [pathArray, undefined]; + }, + [[] as string[], input as unknown], + ); +} + +function applyFieldsFilter(entity: Entity, fields?: string[]): Entity { + if (!fields?.length) { + return entity; + } + const output: Record = {}; + for (const field of fields) { + const [pathArray, value] = getPathArrayAndValue(entity, field); + if (value !== undefined) { + lodash.set(output, pathArray, value); + } + } + return output as Entity; +} + +function applyOrdering(entities: Entity[], order?: EntityOrderQuery): Entity[] { + if (!order) { + return entities; + } + const orders = [order].flat(); + if (orders.length === 0) { + return entities; + } + + const searchMap = new Map>(); + for (const entity of entities) { + searchMap.set(entity, buildEntitySearch(entity)); + } + + return [...entities].sort((a, b) => { + const aRows = searchMap.get(a)!; + const bRows = searchMap.get(b)!; + + for (const { field, order: dir } of orders) { + const key = field.toLocaleLowerCase('en-US'); + const aRow = aRows.find(r => r.key.toLocaleLowerCase('en-US') === key); + const bRow = bRows.find(r => r.key.toLocaleLowerCase('en-US') === key); + const aValue = + aRow?.value !== null && aRow?.value !== undefined + ? String(aRow.value).toLocaleLowerCase('en-US') + : null; + const bValue = + bRow?.value !== null && bRow?.value !== undefined + ? String(bRow.value).toLocaleLowerCase('en-US') + : null; + + if (aValue === null && bValue === null) continue; + if (aValue === null) return 1; + if (bValue === null) return -1; + + if (aValue < bValue) return dir === 'asc' ? -1 : 1; + if (aValue > bValue) return dir === 'asc' ? 1 : -1; + } + + // Stable tie-breaker by entity ref, matching backend behavior + const aRef = stringifyEntityRef(a); + const bRef = stringifyEntityRef(b); + if (aRef < bRef) return -1; + if (aRef > bRef) return 1; + return 0; + }); +} + +function applyFullTextFilter( + entities: Entity[], + fullTextFilter?: { term: string; fields?: string[] }, +): Entity[] { + if (!fullTextFilter?.term?.trim()) { + return entities; + } + const term = fullTextFilter.term.trim().toLocaleLowerCase('en-US'); + const fields = fullTextFilter.fields?.map(f => f.toLocaleLowerCase('en-US')); + + return entities.filter(entity => { + const rows = buildEntitySearch(entity); + return rows.some(row => { + if ( + fields?.length && + !fields.includes(row.key.toLocaleLowerCase('en-US')) + ) { + return false; + } + if (row.value === null || row.value === undefined) return false; + const value = String(row.value).toLocaleLowerCase('en-US'); + return value.includes(term); + }); + }); +} + /** - * Implements a VERY basic fake catalog client that stores entities in memory. - * It has severely limited functionality, and is only useful under certain - * circumstances in tests. + * Implements a fake catalog client that stores entities in memory. + * Supports filtering, ordering, pagination, full-text search, and field + * projection for entity query methods. Location and validation methods + * throw {@link @backstage/errors#NotImplementedError}. * * @public */ @@ -148,7 +320,34 @@ export class InMemoryCatalogClient implements CatalogApi { request?: GetEntitiesRequest, ): Promise { const filter = createFilter(request?.filter); - return { items: this.#entities.filter(filter) }; + let items = this.#entities.filter(filter); + items = applyOrdering(items, request?.order); + + let offset = request?.offset ?? 0; + let limit = request?.limit; + + if (request?.after) { + try { + const cursor = parseCursor(request.after); + if (cursor.offset !== undefined) offset = cursor.offset as number; + if (cursor.limit !== undefined) limit = cursor.limit as number; + } catch { + throw new InputError('Invalid cursor'); + } + } + + if (offset > 0) { + items = items.slice(offset); + } + if (limit !== undefined) { + items = items.slice(0, limit); + } + + return { + items: request?.fields + ? items.map(e => applyFieldsFilter(e, request.fields)) + : items, + }; } async getEntitiesByRefs( @@ -156,26 +355,104 @@ export class InMemoryCatalogClient implements CatalogApi { ): Promise { const filter = createFilter(request.filter); const refMap = this.#createEntityRefMap(); + const items = request.entityRefs + .map(ref => refMap.get(ref)) + .map(e => (e && filter(e) ? e : undefined)); return { - items: request.entityRefs - .map(ref => refMap.get(ref)) - .map(e => (e && filter(e) ? e : undefined)), + items: request.fields + ? items.map(e => (e ? applyFieldsFilter(e, request.fields) : undefined)) + : items, }; } async queryEntities( request?: QueryEntitiesRequest, ): Promise { + // Decode query parameters from cursor or from the request directly + let filter: EntityFilterQuery | undefined; + let orderFields: EntityOrderQuery | undefined; + let fullTextFilter: { term: string; fields?: string[] } | undefined; + let offset: number; + let limit: number | undefined; + if (request && 'cursor' in request) { - return { items: [], pageInfo: {}, totalItems: 0 }; + let c: Record; + try { + c = parseCursor(request.cursor); + } catch { + throw new InputError('Invalid cursor'); + } + filter = deserializeFilter(c.filter as any[]); + orderFields = c.orderFields as EntityOrderQuery | undefined; + fullTextFilter = c.fullTextFilter as typeof fullTextFilter; + offset = c.offset as number; + limit = request.limit; + } else { + filter = request?.filter; + orderFields = request?.orderFields; + fullTextFilter = request?.fullTextFilter; + offset = request?.offset ?? 0; + limit = request?.limit; } - const filter = createFilter(request?.filter); - const items = this.#entities.filter(filter); - // TODO(Rugvip): Pagination + + // Apply filter + let items = this.#entities.filter(createFilter(filter)); + + // Apply full-text filter, defaulting to the sort field or metadata.uid + if (fullTextFilter) { + const orderFieldsList = orderFields ? [orderFields].flat() : []; + items = applyFullTextFilter(items, { + ...fullTextFilter, + fields: fullTextFilter.fields ?? [ + orderFieldsList[0]?.field ?? 'metadata.uid', + ], + }); + } + + items = applyOrdering(items, orderFields); + + const totalItems = items.length; + + // No pagination requested, return all items + if (limit === undefined && offset === 0) { + return { + items: request?.fields + ? items.map(e => applyFieldsFilter(e, request?.fields)) + : items, + pageInfo: {}, + totalItems, + }; + } + + const effectiveLimit = limit ?? totalItems; + const pageItems = items.slice(offset, offset + effectiveLimit); + + const cursorBase = { + filter: serializeFilter(filter), + orderFields, + fullTextFilter, + totalItems, + }; + const pageInfo: QueryEntitiesResponse['pageInfo'] = {}; + if (offset + effectiveLimit < totalItems) { + pageInfo.nextCursor = makeCursor({ + ...cursorBase, + offset: offset + effectiveLimit, + }); + } + if (offset > 0) { + pageInfo.prevCursor = makeCursor({ + ...cursorBase, + offset: Math.max(0, offset - effectiveLimit), + }); + } + return { - items, - pageInfo: {}, - totalItems: items.length, + items: request?.fields + ? pageItems.map(e => applyFieldsFilter(e, request?.fields)) + : pageItems, + totalItems, + pageInfo, }; } @@ -219,14 +496,21 @@ export class InMemoryCatalogClient implements CatalogApi { const facetValues = new Map(); for (const entity of filteredEntities) { const rows = buildEntitySearch(entity); - const value = rows.find( - row => row.key === facet.toLocaleLowerCase('en-US'), - )?.value; - if (value) { - facetValues.set( - String(value), - (facetValues.get(String(value)) ?? 0) + 1, - ); + // Use a Set to count each distinct value once per entity, + // matching the backend's count(DISTINCT entity_id) behavior + const uniqueValues = new Set( + rows + .filter( + row => + row.key.toLocaleLowerCase('en-US') === + facet.toLocaleLowerCase('en-US'), + ) + .map(row => row.value) + .filter(v => v !== null && v !== undefined) + .map(v => String(v)), + ); + for (const value of uniqueValues) { + facetValues.set(value, (facetValues.get(value) ?? 0) + 1); } } const counts = Array.from(facetValues.entries()).map( diff --git a/yarn.lock b/yarn.lock index 61cdfd6deb..b83cf8ef4e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3218,7 +3218,9 @@ __metadata: "@backstage/cli": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" + "@types/lodash": "npm:^4.14.151" cross-fetch: "npm:^4.0.0" + lodash: "npm:^4.17.21" msw: "npm:^1.0.0" uri-template: "npm:^2.0.0" languageName: unknown