Merge pull request #25357 from BethGriggs/batch-requests

fix(plugin-org): implement batching in useGetEntities to handle large…

.changeset/fast-pens-smell.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-org': patch
+---
+
+The `useGetEntities` hook could result in requests to `/api/catalog/entities` where the headers exceed the default maximum Node.js header size of 16KB. The hook logic has been adjusted to batch the requests.

plugins/org/src/components/Cards/OwnershipCard/useGetEntities.test.ts

@@ -231,6 +231,114 @@ describe('useGetEntities', () => {
       );
     });
   });
+
+  describe('useGetEntities with 500+ relations', () => {
+    const manyGroups = Array.from({ length: 555 }, (_, i) =>
+      createGroupEntityFromName(`group${i + 1}`),
+    );
+
+    const givenUserWithManyGroups = {
+      kind: 'User',
+      metadata: {
+        name: givenUser,
+      },
+      spec: {
+        memberOf: manyGroups.map(
+          group => `group:default/${group.metadata.name}`,
+        ),
+      },
+    } as Partial<Entity> as Entity;
+
+    beforeEach(() => {
+      getEntityRelationsMock.mockImplementation(entity =>
+        entity?.kind === 'User'
+          ? manyGroups.map(group => createGroupRefFromName(group.metadata.name))
+          : [],
+      );
+      (catalogApiMock.getEntities as jest.Mock).mockClear();
+    });
+
+    it('should handle 500+ relations without exceeding URL length limits', async () => {
+      const { result } = renderHook(
+        ({ entity }) => useGetEntities(entity, 'aggregated'),
+        {
+          initialProps: { entity: givenUserWithManyGroups },
+        },
+      );
+
+      await waitFor(() => expect(result.current.loading).toBe(false), {
+        timeout: 5000,
+      });
+
+      const callArgs = (catalogApiMock.getEntities as jest.Mock).mock
+        .calls[0][0];
+
+      expect(
+        callArgs.filter[0]['relations.ownedBy'].length,
+      ).toBeLessThanOrEqual(100);
+
+      const owners = callArgs.filter[0]['relations.ownedBy'];
+
+      expect(Array.isArray(owners)).toBeTruthy();
+      expect(owners.length).toBeLessThanOrEqual(100);
+    });
+  });
+
+  describe('useGetEntities exceeding default 16384 bytes header size', () => {
+    const largeNumberOfGroups = Array.from({ length: 600 }, (_, i) =>
+      createGroupEntityFromName(`very-long-group-name-${i + 1}`),
+    );
+
+    const givenUserWithLargeNumberOfGroups = {
+      kind: 'User',
+      metadata: {
+        name: givenUser,
+      },
+      spec: {
+        memberOf: largeNumberOfGroups.map(
+          group => `group:default/${group.metadata.name}`,
+        ),
+      },
+    } as Partial<Entity> as Entity;
+
+    beforeEach(() => {
+      getEntityRelationsMock.mockImplementation(entity =>
+        entity?.kind === 'User'
+          ? largeNumberOfGroups.map(group =>
+              createGroupRefFromName(group.metadata.name),
+            )
+          : [],
+      );
+      (catalogApiMock.getEntities as jest.Mock).mockClear();
+    });
+
+    it('should batch the request to avoid exceeding header size limits', async () => {
+      const { result } = renderHook(
+        ({ entity }) => useGetEntities(entity, 'aggregated'),
+        {
+          initialProps: { entity: givenUserWithLargeNumberOfGroups },
+        },
+      );
+
+      await waitFor(() => expect(result.current.loading).toBe(false), {
+        timeout: 5000,
+      });
+      const callArgs = (catalogApiMock.getEntities as jest.Mock).mock
+        .calls[0][0];
+
+      const url = new URL(
+        `http://localhost/api/catalog/entities?${new URLSearchParams(
+          callArgs as any,
+        )}`,
+      );
+      const headerSize = url.href.length;
+      expect(headerSize).toBeLessThanOrEqual(16384);
+
+      const owners = callArgs.filter[0]['relations.ownedBy'];
+      expect(Array.isArray(owners)).toBeTruthy();
+      expect(owners.length).toBeLessThanOrEqual(100);
+    });
+  });
 });
 
 function createGroupEntityFromName(name: string): Entity {

plugins/org/src/components/Cards/OwnershipCard/useGetEntities.ts

@@ -32,7 +32,7 @@ import { useApi } from '@backstage/core-plugin-api';
 import useAsync from 'react-use/esm/useAsync';
 import qs from 'qs';
 import { EntityRelationAggregation } from '../types';
-import { uniq } from 'lodash';
+import { uniq, uniqBy } from 'lodash';
 
 const limiter = limiterFactory(5);
 
@@ -143,26 +143,42 @@ const getOwners = async (
   return [stringifyEntityRef(entity)];
 };
 
-const getOwnedEntitiesByOwners = (
+const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+const batchGetOwnedEntitiesByOwners = async (
   owners: string[],
   kinds: string[],
   catalogApi: CatalogApi,
-) =>
-  catalogApi.getEntities({
-    filter: [
-      {
-        kind: kinds,
-        'relations.ownedBy': owners,
-      },
-    ],
-    fields: [
-      'kind',
-      'metadata.name',
-      'metadata.namespace',
-      'spec.type',
-      'relations',
-    ],
-  });
+  batchSize: number = 100,
+  delayMs: number = 100,
+) => {
+  const results = [];
+
+  for (let i = 0; i < owners.length; i += batchSize) {
+    const batch = owners.slice(i, i + batchSize);
+    const response = await catalogApi.getEntities({
+      filter: [
+        {
+          kind: kinds,
+          'relations.ownedBy': batch,
+        },
+      ],
+      fields: [
+        'kind',
+        'metadata.name',
+        'metadata.namespace',
+        'spec.type',
+        'relations',
+      ],
+    });
+
+    results.push(...response.items);
+
+    if (i + batchSize < owners.length) await delay(delayMs);
+  }
+
+  return uniqBy(results, stringifyEntityRef);
+};
 
 export function useGetEntities(
   entity: Entity,
@@ -191,13 +207,13 @@ export function useGetEntities(
   } = useAsync(async () => {
     const owners = await getOwners(entity, relationAggregation, catalogApi);
 
-    const ownedEntitiesList = await getOwnedEntitiesByOwners(
+    const ownedEntitiesList = await batchGetOwnedEntitiesByOwners(
       owners,
       kinds,
       catalogApi,
     );
 
-    const counts = ownedEntitiesList.items.reduce(
+    const counts = ownedEntitiesList.reduce(
       (acc: EntityTypeProps[], ownedEntity) => {
         const match = acc.find(
           x => x.kind === ownedEntity.kind && x.type === ownedEntity.spec?.type,