diff --git a/.changeset/olive-moons-burn.md b/.changeset/olive-moons-burn.md new file mode 100644 index 0000000000..625d97c5a7 --- /dev/null +++ b/.changeset/olive-moons-burn.md @@ -0,0 +1,24 @@ +--- +'@backstage/catalog-client': minor +'@backstage/plugin-catalog-react': minor +'@backstage/plugin-catalog-node': minor +--- + +Introduced new `streamEntities` async generator method for the catalog. + +Catalog API and Catalog Service now includes a `streamEntities` method that allows for streaming entities from the catalog. +This method is designed to handle large datasets efficiently by processing entities in a stream rather than loading them +all into memory at once. This is useful when you need to fetch a large number of entities but do not want to use pagination +or fetch all entities at once. + +Example usage: + +```ts +const pageStream = catalogClient.streamEntities({ pageSize: 100 }, { token }); +for await (const page of pageStream) { + // Handle page of entities + for (const entity of page) { + console.log(entity); + } +} +``` diff --git a/packages/catalog-client/report-testUtils.api.md b/packages/catalog-client/report-testUtils.api.md index f5bcf55c53..b96c4f01ca 100644 --- a/packages/catalog-client/report-testUtils.api.md +++ b/packages/catalog-client/report-testUtils.api.md @@ -22,6 +22,7 @@ import { GetLocationsResponse } from '@backstage/catalog-client'; import { Location as Location_2 } from '@backstage/catalog-client'; import { QueryEntitiesRequest } from '@backstage/catalog-client'; import { QueryEntitiesResponse } from '@backstage/catalog-client'; +import { StreamEntitiesRequest } from '@backstage/catalog-client'; import { ValidateEntityResponse } from '@backstage/catalog-client'; // @public @@ -70,6 +71,8 @@ export class InMemoryCatalogClient implements CatalogApi { // (undocumented) removeLocationById(_id: string): Promise; // (undocumented) + streamEntities(request?: StreamEntitiesRequest): AsyncIterable; + // (undocumented) validateEntity( _entity: Entity, _locationRef: string, diff --git a/packages/catalog-client/report.api.md b/packages/catalog-client/report.api.md index ebb0f5ac97..452d788ba9 100644 --- a/packages/catalog-client/report.api.md +++ b/packages/catalog-client/report.api.md @@ -88,6 +88,10 @@ export interface CatalogApi { id: string, options?: CatalogRequestOptions, ): Promise; + streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogRequestOptions, + ): AsyncIterable; validateEntity( entity: Entity, locationRef: string, @@ -170,6 +174,10 @@ export class CatalogClient implements CatalogApi { id: string, options?: CatalogRequestOptions, ): Promise; + streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogRequestOptions, + ): AsyncIterable; validateEntity( entity: Entity, locationRef: string, @@ -317,6 +325,14 @@ export type QueryEntitiesResponse = { }; }; +// @public +export type StreamEntitiesRequest = Omit< + QueryEntitiesInitialRequest, + 'limit' | 'offset' +> & { + pageSize?: number; +}; + // @public export type ValidateEntityResponse = | { diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index abafeb18a5..8307f77cdd 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -540,6 +540,71 @@ describe('CatalogClient', () => { }); }); + describe('streamEntities', () => { + const defaultResponse: QueryEntitiesResponse = { + items: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + pageInfo: { + nextCursor: 'next', + prevCursor: 'prev', + }, + totalItems: 10, + }; + + beforeEach(() => { + server.use( + rest.get(`${mockBaseUrl}/entities/by-query`, (req, res, ctx) => { + const cursor = req.url.searchParams.get('cursor'); + if (cursor === 'next') { + return res(ctx.json({ items: [], pageInfo: {}, totalItems: 0 })); + } + return res(ctx.json(defaultResponse)); + }), + ); + }); + + it('should stream entities', async () => { + const stream = client.streamEntities({}, { token }); + const results: Entity[][] = []; + for await (const entityPage of stream) { + results.push(entityPage); + } + expect(results).toEqual([defaultResponse.items, []]); + }); + + it('should handle errors', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => res(ctx.status(401))); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + const stream = client.streamEntities({}, { token }); + await expect(async () => { + const results = []; + for await (const entity of stream) { + results.push(entity); + } + }).rejects.toThrow(/Request failed with 401 Unauthorized/); + }); + }); + describe('getEntityByRef', () => { const existingEntity: Entity = { apiVersion: 'v1', diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 7d86eed95d..07e6ad2946 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -40,6 +40,7 @@ import { Location, QueryEntitiesRequest, QueryEntitiesResponse, + StreamEntitiesRequest, ValidateEntityResponse, } from './types/api'; import { isQueryEntitiesInitialRequest, splitRefsIntoChunks } from './utils'; @@ -48,6 +49,7 @@ import type { AnalyzeLocationRequest, AnalyzeLocationResponse, } from '@backstage/plugin-catalog-common'; +import { DEFAULT_STREAM_ENTITIES_LIMIT } from './constants.ts'; /** * A frontend and backend compatible client for communicating with the Backstage @@ -456,6 +458,27 @@ export class CatalogClient implements CatalogApi { return response.json() as Promise; } + /** + * {@inheritdoc CatalogApi.streamEntities} + */ + async *streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogRequestOptions, + ): AsyncIterable { + let cursor: string | undefined = undefined; + const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT; + do { + const res = await this.queryEntities( + cursor ? { ...request, cursor, limit } : { ...request, limit }, + options, + ); + + yield res.items; + + cursor = res.pageInfo.nextCursor; + } while (cursor); + } + // // Private methods // diff --git a/packages/catalog-client/src/constants.ts b/packages/catalog-client/src/constants.ts new file mode 100644 index 0000000000..dd8d0530df --- /dev/null +++ b/packages/catalog-client/src/constants.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const DEFAULT_STREAM_ENTITIES_LIMIT = 500; diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts index c5d1f3abc8..dbe3390747 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts @@ -123,6 +123,16 @@ describe('InMemoryCatalogClient', () => { }); }); + it('streamEntities', async () => { + const client = new InMemoryCatalogClient({ entities }); + const stream = client.streamEntities(); + const results: Entity[][] = []; + for await (const page of stream) { + results.push(page); + } + expect(results).toEqual([entities]); + }); + it('getEntityAncestors', async () => { const client = new InMemoryCatalogClient({ entities }); await expect( diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts index 97663de922..a9db99ab77 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts @@ -32,6 +32,7 @@ import { Location, QueryEntitiesRequest, QueryEntitiesResponse, + StreamEntitiesRequest, ValidateEntityResponse, } from '@backstage/catalog-client'; import { @@ -48,6 +49,7 @@ import type { AnalyzeLocationRequest, AnalyzeLocationResponse, } from '@backstage/plugin-catalog-common'; +import { DEFAULT_STREAM_ENTITIES_LIMIT } from '../constants.ts'; function buildEntitySearch(entity: Entity) { const rows = traverse(entity); @@ -279,6 +281,22 @@ export class InMemoryCatalogClient implements CatalogApi { throw new NotImplementedError('Method not implemented.'); } + async *streamEntities( + request?: StreamEntitiesRequest, + ): AsyncIterable { + let cursor: string | undefined = undefined; + const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT; + do { + const res = await this.queryEntities( + cursor ? { ...request, limit, cursor } : { ...request, limit }, + ); + + yield res.items; + + cursor = res.pageInfo.nextCursor; + } while (cursor); + } + #createEntityRefMap() { return new Map(this.#entities.map(e => [stringifyEntityRef(e), e])); } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 28fe8bf276..014baba613 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -465,6 +465,21 @@ export type QueryEntitiesResponse = { }; }; +/** + * Stream entities request for {@link CatalogClient.streamEntities}. + * + * @public + */ +export type StreamEntitiesRequest = Omit< + QueryEntitiesInitialRequest, + 'limit' | 'offset' +> & { + /** + * The number of entities to fetch in each page. Defaults to 500. + */ + pageSize?: number; +}; + /** * A client for interacting with the Backstage software catalog through its API. * @@ -692,4 +707,18 @@ export interface CatalogApi { location: AnalyzeLocationRequest, options?: CatalogRequestOptions, ): Promise; + + /** + * Asynchronously streams entities from the catalog. Uses `queryEntities` + * to fetch entities in batches, and yields them one page at a time. + * + * @public + * + * @param request - Request parameters + * @param options - Additional options + */ + streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogRequestOptions, + ): AsyncIterable; } diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 73b16f4554..5b483d9b29 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -38,5 +38,6 @@ export type { QueryEntitiesInitialRequest, QueryEntitiesRequest, QueryEntitiesResponse, + StreamEntitiesRequest, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/plugins/catalog-node/report-testUtils.api.md b/plugins/catalog-node/report-testUtils.api.md index 3fa7057a99..d687ff387b 100644 --- a/plugins/catalog-node/report-testUtils.api.md +++ b/plugins/catalog-node/report-testUtils.api.md @@ -27,6 +27,7 @@ import { QueryEntitiesRequest } from '@backstage/catalog-client'; import { QueryEntitiesResponse } from '@backstage/catalog-client'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceMock } from '@backstage/backend-test-utils'; +import { StreamEntitiesRequest } from '@backstage/catalog-client'; import { ValidateEntityResponse } from '@backstage/catalog-client'; // @public @@ -107,6 +108,11 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi { options?: CatalogServiceRequestOptions | CatalogRequestOptions, ): Promise; // (undocumented) + streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogServiceRequestOptions | CatalogRequestOptions, + ): AsyncIterable; + // (undocumented) validateEntity( entity: Entity, locationRef: string, diff --git a/plugins/catalog-node/report.api.md b/plugins/catalog-node/report.api.md index debafad5c4..25ab9145ef 100644 --- a/plugins/catalog-node/report.api.md +++ b/plugins/catalog-node/report.api.md @@ -27,6 +27,7 @@ import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common import { QueryEntitiesRequest } from '@backstage/catalog-client'; import { QueryEntitiesResponse } from '@backstage/catalog-client'; import { ServiceRef } from '@backstage/backend-plugin-api'; +import { StreamEntitiesRequest } from '@backstage/catalog-client'; import { ValidateEntityResponse } from '@backstage/catalog-client'; // @public (undocumented) @@ -195,6 +196,11 @@ export interface CatalogService { options: CatalogServiceRequestOptions, ): Promise; // (undocumented) + streamEntities( + request: StreamEntitiesRequest | undefined, + options: CatalogServiceRequestOptions, + ): AsyncIterable; + // (undocumented) validateEntity( entity: Entity, locationRef: string, diff --git a/plugins/catalog-node/src/catalogService.ts b/plugins/catalog-node/src/catalogService.ts index 9478c6b48a..a6ecedd5c5 100644 --- a/plugins/catalog-node/src/catalogService.ts +++ b/plugins/catalog-node/src/catalogService.ts @@ -15,11 +15,11 @@ */ import { + AuthService, + BackstageCredentials, + coreServices, createServiceFactory, createServiceRef, - coreServices, - BackstageCredentials, - AuthService, } from '@backstage/backend-plugin-api'; import { AddLocationRequest, @@ -39,6 +39,7 @@ import { Location, QueryEntitiesRequest, QueryEntitiesResponse, + StreamEntitiesRequest, ValidateEntityResponse, } from '@backstage/catalog-client'; import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; @@ -141,6 +142,11 @@ export interface CatalogService { location: AnalyzeLocationRequest, options: CatalogServiceRequestOptions, ): Promise; + + streamEntities( + request: StreamEntitiesRequest | undefined, + options: CatalogServiceRequestOptions, + ): AsyncIterable; } class DefaultCatalogService implements CatalogService { @@ -320,6 +326,16 @@ class DefaultCatalogService implements CatalogService { ); } + async *streamEntities( + request: StreamEntitiesRequest | undefined, + options: CatalogServiceRequestOptions, + ): AsyncIterable { + yield* this.#catalogApi.streamEntities( + request, + await this.#getOptions(options), + ); + } + async #getOptions( options: CatalogServiceRequestOptions, ): Promise { diff --git a/plugins/catalog-node/src/testUtils/catalogServiceMock.ts b/plugins/catalog-node/src/testUtils/catalogServiceMock.ts index 20e04e12b4..59c66c0b88 100644 --- a/plugins/catalog-node/src/testUtils/catalogServiceMock.ts +++ b/plugins/catalog-node/src/testUtils/catalogServiceMock.ts @@ -15,9 +15,9 @@ */ import { + createServiceFactory, ServiceFactory, ServiceRef, - createServiceFactory, } from '@backstage/backend-plugin-api'; import { InMemoryCatalogClient } from '@backstage/catalog-client/testUtils'; import { Entity } from '@backstage/catalog-model'; @@ -103,5 +103,6 @@ export namespace catalogServiceMock { getLocationByEntity: jest.fn(), validateEntity: jest.fn(), analyzeLocation: jest.fn(), + streamEntities: jest.fn(), })); } diff --git a/plugins/catalog-node/src/testUtils/types.ts b/plugins/catalog-node/src/testUtils/types.ts index 96380e9434..a5bf125cfd 100644 --- a/plugins/catalog-node/src/testUtils/types.ts +++ b/plugins/catalog-node/src/testUtils/types.ts @@ -31,6 +31,7 @@ import { Location, QueryEntitiesRequest, QueryEntitiesResponse, + StreamEntitiesRequest, ValidateEntityResponse, } from '@backstage/catalog-client'; import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; @@ -135,4 +136,9 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi { location: AnalyzeLocationRequest, options?: CatalogServiceRequestOptions | CatalogRequestOptions, ): Promise; + + streamEntities( + request?: StreamEntitiesRequest, + options?: CatalogServiceRequestOptions | CatalogRequestOptions, + ): AsyncIterable; } diff --git a/plugins/catalog-react/src/testUtils/catalogApiMock.ts b/plugins/catalog-react/src/testUtils/catalogApiMock.ts index 42c6b1004f..a37d2669ad 100644 --- a/plugins/catalog-react/src/testUtils/catalogApiMock.ts +++ b/plugins/catalog-react/src/testUtils/catalogApiMock.ts @@ -102,5 +102,7 @@ export namespace catalogApiMock { getLocationByEntity: jest.fn(), validateEntity: jest.fn(), analyzeLocation: jest.fn(), + streamEntities: jest.fn(), + streamEntityPages: jest.fn(), })); }