From 81c54d1f25255af82a4060d5d71ea61f7f51379d Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 12:07:48 +0100 Subject: [PATCH 1/6] Fetch useRelatedEntities in batches Rather than making a request per relation, filter multiple relations in a single request. To avoid the query string getting very large, make multiple requests on bacthes of 100 relations. Signed-off-by: Will Sewell --- .changeset/friendly-readers-promise.md | 5 ++++ .../src/hooks/useRelatedEntities.ts | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 .changeset/friendly-readers-promise.md diff --git a/.changeset/friendly-readers-promise.md b/.changeset/friendly-readers-promise.md new file mode 100644 index 0000000000..30607d5db2 --- /dev/null +++ b/.changeset/friendly-readers-promise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Fetch relations in batches in `useRelatedEntities` diff --git a/plugins/catalog-react/src/hooks/useRelatedEntities.ts b/plugins/catalog-react/src/hooks/useRelatedEntities.ts index baf6cf5fc7..15182d3c2f 100644 --- a/plugins/catalog-react/src/hooks/useRelatedEntities.ts +++ b/plugins/catalog-react/src/hooks/useRelatedEntities.ts @@ -18,6 +18,8 @@ import { useApi } from '@backstage/core'; import { useAsync } from 'react-use'; import { catalogApiRef } from '../api'; +const BATCH_SIZE = 100; + export function useRelatedEntities( entity: Entity, { type, kind }: { type?: string; kind?: string }, @@ -40,16 +42,28 @@ export function useRelatedEntities( return []; } - // TODO: This code could be more efficient if there was an endpoint in the - // backend that either returns the relations of entity (filtered by type) - // or if there is a way to perform a batch request by entity name. However, - // such an implementation would probably be better placed in the graphql API. + // Make requests in separate batches to limit query string size + // (there is a `filter` param for each relation) + const relationBatches = []; + for (let i = 0; i < relations.length; i += BATCH_SIZE) { + relationBatches.push(relations.slice(i, i + BATCH_SIZE)); + } + const results = await Promise.all( - relations?.map(r => catalogApi.getEntityByName(r.target)), + relationBatches.map(batch => { + return catalogApi.getEntities({ + filter: batch.map(({ target }) => { + return { + kind: target.kind, + 'metadata.name': target.name, + 'metadata.namespace': target.namespace, + }; + }), + }); + }), ); - // Skip entities that where not found, for example if a relation references - // an entity that doesn't exist. - return results.filter(e => e) as Entity[]; + + return results.map(r => r.items).flat(); }, [entity, type]); return { From e763b7b156c0b67ff7c7faa1d55e444414553b4d Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 13:36:22 +0100 Subject: [PATCH 2/6] Reduce batch size to 20 Signed-off-by: Will Sewell --- plugins/catalog-react/src/hooks/useRelatedEntities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-react/src/hooks/useRelatedEntities.ts b/plugins/catalog-react/src/hooks/useRelatedEntities.ts index 15182d3c2f..5649a90f52 100644 --- a/plugins/catalog-react/src/hooks/useRelatedEntities.ts +++ b/plugins/catalog-react/src/hooks/useRelatedEntities.ts @@ -18,7 +18,7 @@ import { useApi } from '@backstage/core'; import { useAsync } from 'react-use'; import { catalogApiRef } from '../api'; -const BATCH_SIZE = 100; +const BATCH_SIZE = 20; export function useRelatedEntities( entity: Entity, From 33c21766a9b2b80ba02f550be58b7d60a6205119 Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 15:54:37 +0100 Subject: [PATCH 3/6] Add lodash to catalog-react Signed-off-by: Will Sewell --- plugins/catalog-react/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-react/package.json b/plugins/catalog-react/package.json index 91201a49a1..57a9479a3c 100644 --- a/plugins/catalog-react/package.json +++ b/plugins/catalog-react/package.json @@ -33,6 +33,7 @@ "@backstage/core": "^0.7.3", "@material-ui/core": "^4.11.0", "@types/react": "^16.9", + "lodash": "^4.17.15", "react": "^16.13.1", "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", From cf3faccd846835da5883b66687750b4666b40ea0 Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 15:58:33 +0100 Subject: [PATCH 4/6] Use lodash `chunk` to create batches Signed-off-by: Will Sewell --- plugins/catalog-react/src/hooks/useRelatedEntities.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-react/src/hooks/useRelatedEntities.ts b/plugins/catalog-react/src/hooks/useRelatedEntities.ts index 5649a90f52..25d96626d7 100644 --- a/plugins/catalog-react/src/hooks/useRelatedEntities.ts +++ b/plugins/catalog-react/src/hooks/useRelatedEntities.ts @@ -15,6 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; import { useApi } from '@backstage/core'; +import { chunk } from 'lodash'; import { useAsync } from 'react-use'; import { catalogApiRef } from '../api'; @@ -44,10 +45,7 @@ export function useRelatedEntities( // Make requests in separate batches to limit query string size // (there is a `filter` param for each relation) - const relationBatches = []; - for (let i = 0; i < relations.length; i += BATCH_SIZE) { - relationBatches.push(relations.slice(i, i + BATCH_SIZE)); - } + const relationBatches = chunk(relations, BATCH_SIZE); const results = await Promise.all( relationBatches.map(batch => { From fd9822432d86b47a62a6a156c04a64dbf44d7785 Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 17:21:57 +0100 Subject: [PATCH 5/6] Group relations by kind and namespace Signed-off-by: Will Sewell --- .../src/hooks/useRelatedEntities.ts | 55 ++++++++++++++----- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/plugins/catalog-react/src/hooks/useRelatedEntities.ts b/plugins/catalog-react/src/hooks/useRelatedEntities.ts index 25d96626d7..b47d941ff6 100644 --- a/plugins/catalog-react/src/hooks/useRelatedEntities.ts +++ b/plugins/catalog-react/src/hooks/useRelatedEntities.ts @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; +import { Entity, EntityRelation } from '@backstage/catalog-model'; import { useApi } from '@backstage/core'; -import { chunk } from 'lodash'; +import { chunk, groupBy } from 'lodash'; import { useAsync } from 'react-use'; import { catalogApiRef } from '../api'; @@ -43,25 +43,50 @@ export function useRelatedEntities( return []; } - // Make requests in separate batches to limit query string size - // (there is a `filter` param for each relation) - const relationBatches = chunk(relations, BATCH_SIZE); + // Group the relations by kind and namespace to reduce the size of the request query string. + // Without this grouping, the kind and namespace would need to be specified for each relation, e.g. + // `filter=kind=component,namespace=default,name=example1&filter=kind=component,namespace=default,name=example2` + // with grouping, we can generate a query a string like + // `filter=kind=component,namespace=default,name=example1,example2` + const relationsByKindAndNamespace: EntityRelation[][] = Object.values( + groupBy(relations, ({ target }) => { + return `${target.kind}:${target.namespace}`.toLowerCase(); + }), + ); + + // Split the names within each group into batches to further reduce the query string length. + const batchedRelationsByKindAndNamespace: { + kind: string; + namespace: string; + nameBatches: string[][]; + }[] = []; + for (const rs of relationsByKindAndNamespace) { + batchedRelationsByKindAndNamespace.push({ + // All relations in a group have the same kind and namespace, so its arbitrary which we pick + kind: rs[0].target.kind, + namespace: rs[0].target.namespace, + nameBatches: chunk( + rs.map(r => r.target.name), + BATCH_SIZE, + ), + }); + } const results = await Promise.all( - relationBatches.map(batch => { - return catalogApi.getEntities({ - filter: batch.map(({ target }) => { - return { - kind: target.kind, - 'metadata.name': target.name, - 'metadata.namespace': target.namespace, - }; - }), + batchedRelationsByKindAndNamespace.flatMap(rs => { + return rs.nameBatches.map(names => { + return catalogApi.getEntities({ + filter: { + kind: rs.kind, + 'metadata.namespace': rs.namespace, + 'metadata.name': names, + }, + }); }); }), ); - return results.map(r => r.items).flat(); + return results.flatMap(r => r.items); }, [entity, type]); return { From 4dcab641852281818cabc669c0cb693c975efde2 Mon Sep 17 00:00:00 2001 From: Will Sewell Date: Mon, 3 May 2021 18:49:15 +0100 Subject: [PATCH 6/6] Update test mocks to expect call to `getEntities` Signed-off-by: Will Sewell --- .../ApisCards/ConsumedApisCard.test.tsx | 20 +++++++++++-------- .../components/ApisCards/HasApisCard.test.tsx | 20 +++++++++++-------- .../ApisCards/ProvidedApisCard.test.tsx | 20 +++++++++++-------- .../ConsumingComponentsCard.test.tsx | 20 +++++++++++-------- .../ProvidingComponentsCard.test.tsx | 20 +++++++++++-------- .../HasComponentsCard.test.tsx | 20 +++++++++++-------- .../HasSubcomponentsCard.test.tsx | 20 +++++++++++-------- .../HasSystemsCard/HasSystemsCard.test.tsx | 20 +++++++++++-------- 8 files changed, 96 insertions(+), 64 deletions(-) diff --git a/plugins/api-docs/src/components/ApisCards/ConsumedApisCard.test.tsx b/plugins/api-docs/src/components/ApisCards/ConsumedApisCard.test.tsx index b157e898ca..fa956cd042 100644 --- a/plugins/api-docs/src/components/ApisCards/ConsumedApisCard.test.tsx +++ b/plugins/api-docs/src/components/ApisCards/ConsumedApisCard.test.tsx @@ -96,14 +96,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'API', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'API', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/api-docs/src/components/ApisCards/HasApisCard.test.tsx b/plugins/api-docs/src/components/ApisCards/HasApisCard.test.tsx index 5db3ec2b48..e721f6e918 100644 --- a/plugins/api-docs/src/components/ApisCards/HasApisCard.test.tsx +++ b/plugins/api-docs/src/components/ApisCards/HasApisCard.test.tsx @@ -96,14 +96,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'API', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'API', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/api-docs/src/components/ApisCards/ProvidedApisCard.test.tsx b/plugins/api-docs/src/components/ApisCards/ProvidedApisCard.test.tsx index bb9b2e60d9..67d95605e4 100644 --- a/plugins/api-docs/src/components/ApisCards/ProvidedApisCard.test.tsx +++ b/plugins/api-docs/src/components/ApisCards/ProvidedApisCard.test.tsx @@ -96,14 +96,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'API', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'API', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/api-docs/src/components/ComponentsCards/ConsumingComponentsCard.test.tsx b/plugins/api-docs/src/components/ComponentsCards/ConsumingComponentsCard.test.tsx index 7944b3f90a..104b360ff6 100644 --- a/plugins/api-docs/src/components/ComponentsCards/ConsumingComponentsCard.test.tsx +++ b/plugins/api-docs/src/components/ComponentsCards/ConsumingComponentsCard.test.tsx @@ -101,14 +101,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'Component', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/api-docs/src/components/ComponentsCards/ProvidingComponentsCard.test.tsx b/plugins/api-docs/src/components/ComponentsCards/ProvidingComponentsCard.test.tsx index ab1751b9ba..c7c6bacc84 100644 --- a/plugins/api-docs/src/components/ComponentsCards/ProvidingComponentsCard.test.tsx +++ b/plugins/api-docs/src/components/ComponentsCards/ProvidingComponentsCard.test.tsx @@ -101,14 +101,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'Component', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/catalog/src/components/HasComponentsCard/HasComponentsCard.test.tsx b/plugins/catalog/src/components/HasComponentsCard/HasComponentsCard.test.tsx index 58df0a53f8..8e0a61b05a 100644 --- a/plugins/catalog/src/components/HasComponentsCard/HasComponentsCard.test.tsx +++ b/plugins/catalog/src/components/HasComponentsCard/HasComponentsCard.test.tsx @@ -91,14 +91,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'Component', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/catalog/src/components/HasSubcomponentsCard/HasSubcomponentsCard.test.tsx b/plugins/catalog/src/components/HasSubcomponentsCard/HasSubcomponentsCard.test.tsx index 79b69d35ff..9ad69c9a0c 100644 --- a/plugins/catalog/src/components/HasSubcomponentsCard/HasSubcomponentsCard.test.tsx +++ b/plugins/catalog/src/components/HasSubcomponentsCard/HasSubcomponentsCard.test.tsx @@ -91,14 +91,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'Component', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp( diff --git a/plugins/catalog/src/components/HasSystemsCard/HasSystemsCard.test.tsx b/plugins/catalog/src/components/HasSystemsCard/HasSystemsCard.test.tsx index 08923a59c8..9b7fb0adae 100644 --- a/plugins/catalog/src/components/HasSystemsCard/HasSystemsCard.test.tsx +++ b/plugins/catalog/src/components/HasSystemsCard/HasSystemsCard.test.tsx @@ -89,14 +89,18 @@ describe('', () => { }, ], }; - catalogApi.getEntityByName.mockResolvedValue({ - apiVersion: 'v1', - kind: 'System', - metadata: { - name: 'target-name', - namespace: 'my-namespace', - }, - spec: {}, + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'v1', + kind: 'System', + metadata: { + name: 'target-name', + namespace: 'my-namespace', + }, + spec: {}, + }, + ], }); const { getByText } = await renderInTestApp(