From 9118c56520db40477561fc567a3bc45ae30f2923 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Wed, 3 Sep 2025 08:48:10 +0300 Subject: [PATCH] fix(catalog): remove streamEntityPages function and make the streamEntities work as the pages function did earlier Signed-off-by: Hellgren Heikki --- .changeset/olive-moons-burn.md | 21 +++++-------------- .../catalog-client/report-testUtils.api.md | 4 +--- packages/catalog-client/report.api.md | 8 ------- .../catalog-client/src/CatalogClient.test.ts | 9 -------- packages/catalog-client/src/CatalogClient.ts | 15 ------------- .../testUtils/InMemoryCatalogClient.test.ts | 10 --------- .../src/testUtils/InMemoryCatalogClient.ts | 11 ---------- packages/catalog-client/src/types/api.ts | 16 +------------- plugins/catalog-node/report-testUtils.api.md | 5 ----- plugins/catalog-node/report.api.md | 5 ----- plugins/catalog-node/src/catalogService.ts | 17 +-------------- .../src/testUtils/catalogServiceMock.ts | 1 - plugins/catalog-node/src/testUtils/types.ts | 5 ----- 13 files changed, 8 insertions(+), 119 deletions(-) diff --git a/.changeset/olive-moons-burn.md b/.changeset/olive-moons-burn.md index 42c488e637..625d97c5a7 100644 --- a/.changeset/olive-moons-burn.md +++ b/.changeset/olive-moons-burn.md @@ -4,7 +4,7 @@ '@backstage/plugin-catalog-node': minor --- -Introduced new `streamEntities` and `streamEntityPages` async generator methods for the catalog. +Introduced new `streamEntities` async generator method for the catalog. Catalog API and Catalog Service now includes a `streamEntities` method that allows for streaming entities from the catalog. This method is designed to handle large datasets efficiently by processing entities in a stream rather than loading them @@ -14,22 +14,11 @@ or fetch all entities at once. Example usage: ```ts -const stream = catalogClient.streamEntities({}, { token }); -for await (const entity of stream) { - // Handle entity -} -``` - -Additionally, a `streamEntityPages` method is available that streams entities in pages, allowing for batch processing of entities. -This method is more efficient than `streamEntities` when you can process entities in chunks. -Example usage: - -```ts -const pageStream = catalogClient.streamEntityPages( - { pageSize: 100 }, - { token }, -); +const pageStream = catalogClient.streamEntities({ pageSize: 100 }, { token }); for await (const page of pageStream) { // Handle page of entities + for (const entity of page) { + console.log(entity); + } } ``` diff --git a/packages/catalog-client/report-testUtils.api.md b/packages/catalog-client/report-testUtils.api.md index 01bb94214b..b96c4f01ca 100644 --- a/packages/catalog-client/report-testUtils.api.md +++ b/packages/catalog-client/report-testUtils.api.md @@ -71,9 +71,7 @@ export class InMemoryCatalogClient implements CatalogApi { // (undocumented) removeLocationById(_id: string): Promise; // (undocumented) - streamEntities(request?: StreamEntitiesRequest): AsyncIterable; - // (undocumented) - streamEntityPages(request?: StreamEntitiesRequest): AsyncIterable; + streamEntities(request?: StreamEntitiesRequest): AsyncIterable; // (undocumented) validateEntity( _entity: Entity, diff --git a/packages/catalog-client/report.api.md b/packages/catalog-client/report.api.md index 4175cf3a4b..452d788ba9 100644 --- a/packages/catalog-client/report.api.md +++ b/packages/catalog-client/report.api.md @@ -91,10 +91,6 @@ export interface CatalogApi { streamEntities( request?: StreamEntitiesRequest, options?: CatalogRequestOptions, - ): AsyncIterable; - streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogRequestOptions, ): AsyncIterable; validateEntity( entity: Entity, @@ -181,10 +177,6 @@ export class CatalogClient implements CatalogApi { streamEntities( request?: StreamEntitiesRequest, options?: CatalogRequestOptions, - ): AsyncIterable; - streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogRequestOptions, ): AsyncIterable; validateEntity( entity: Entity, diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index e330eefd33..8307f77cdd 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -581,15 +581,6 @@ describe('CatalogClient', () => { it('should stream entities', async () => { const stream = client.streamEntities({}, { token }); - const results: Entity[] = []; - for await (const entity of stream) { - results.push(entity); - } - expect(results).toEqual(defaultResponse.items); - }); - - it('should stream entity pages', async () => { - const stream = client.streamEntityPages({}, { token }); const results: Entity[][] = []; for await (const entityPage of stream) { results.push(entityPage); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index c777c48940..07e6ad2946 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -464,21 +464,6 @@ export class CatalogClient implements CatalogApi { async *streamEntities( request?: StreamEntitiesRequest, options?: CatalogRequestOptions, - ): AsyncIterable { - const pages = this.streamEntityPages(request, options); - for await (const page of pages) { - for (const entity of page) { - yield entity; - } - } - } - - /** - * {@inheritdoc CatalogApi.streamEntityPages} - */ - async *streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogRequestOptions, ): AsyncIterable { let cursor: string | undefined = undefined; const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT; diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts index 35f0a89248..dbe3390747 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts @@ -126,16 +126,6 @@ describe('InMemoryCatalogClient', () => { it('streamEntities', async () => { const client = new InMemoryCatalogClient({ entities }); const stream = client.streamEntities(); - const results: Entity[] = []; - for await (const entity of stream) { - results.push(entity); - } - expect(results).toEqual(entities); - }); - - it('streamEntityPages', async () => { - const client = new InMemoryCatalogClient({ entities }); - const stream = client.streamEntityPages(); const results: Entity[][] = []; for await (const page of stream) { results.push(page); diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts index ef125d3c3a..a9db99ab77 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts @@ -283,17 +283,6 @@ export class InMemoryCatalogClient implements CatalogApi { async *streamEntities( request?: StreamEntitiesRequest, - ): AsyncIterable { - const pages = this.streamEntityPages(request); - for await (const page of pages) { - for (const entity of page) { - yield entity; - } - } - } - - async *streamEntityPages( - request?: StreamEntitiesRequest, ): AsyncIterable { let cursor: string | undefined = undefined; const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 6a83b95d6e..014baba613 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -710,7 +710,7 @@ export interface CatalogApi { /** * Asynchronously streams entities from the catalog. Uses `queryEntities` - * to fetch entities in batches, and yields them one by one. + * to fetch entities in batches, and yields them one page at a time. * * @public * @@ -720,19 +720,5 @@ export interface CatalogApi { streamEntities( request?: StreamEntitiesRequest, options?: CatalogRequestOptions, - ): AsyncIterable; - - /** - * Asynchronously streams entity pages from the catalog. Uses `queryEntities` - * to fetch entities in batches, and yields them one page at a time. - * - * @public - * - * @param request - Request parameters - * @param options - Additional options - */ - streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogRequestOptions, ): AsyncIterable; } diff --git a/plugins/catalog-node/report-testUtils.api.md b/plugins/catalog-node/report-testUtils.api.md index e06fb741d1..d687ff387b 100644 --- a/plugins/catalog-node/report-testUtils.api.md +++ b/plugins/catalog-node/report-testUtils.api.md @@ -111,11 +111,6 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi { streamEntities( request?: StreamEntitiesRequest, options?: CatalogServiceRequestOptions | CatalogRequestOptions, - ): AsyncIterable; - // (undocumented) - streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogServiceRequestOptions | CatalogRequestOptions, ): AsyncIterable; // (undocumented) validateEntity( diff --git a/plugins/catalog-node/report.api.md b/plugins/catalog-node/report.api.md index 9173f89e29..25ab9145ef 100644 --- a/plugins/catalog-node/report.api.md +++ b/plugins/catalog-node/report.api.md @@ -199,11 +199,6 @@ export interface CatalogService { streamEntities( request: StreamEntitiesRequest | undefined, options: CatalogServiceRequestOptions, - ): AsyncIterable; - // (undocumented) - streamEntityPages( - request: StreamEntitiesRequest | undefined, - options: CatalogServiceRequestOptions, ): AsyncIterable; // (undocumented) validateEntity( diff --git a/plugins/catalog-node/src/catalogService.ts b/plugins/catalog-node/src/catalogService.ts index aa6a69a69b..a6ecedd5c5 100644 --- a/plugins/catalog-node/src/catalogService.ts +++ b/plugins/catalog-node/src/catalogService.ts @@ -146,11 +146,6 @@ export interface CatalogService { streamEntities( request: StreamEntitiesRequest | undefined, options: CatalogServiceRequestOptions, - ): AsyncIterable; - - streamEntityPages( - request: StreamEntitiesRequest | undefined, - options: CatalogServiceRequestOptions, ): AsyncIterable; } @@ -334,18 +329,8 @@ class DefaultCatalogService implements CatalogService { async *streamEntities( request: StreamEntitiesRequest | undefined, options: CatalogServiceRequestOptions, - ): AsyncIterable { - yield* this.#catalogApi.streamEntities( - request, - await this.#getOptions(options), - ); - } - - async *streamEntityPages( - request: StreamEntitiesRequest | undefined, - options: CatalogServiceRequestOptions, ): AsyncIterable { - yield* this.#catalogApi.streamEntityPages( + yield* this.#catalogApi.streamEntities( request, await this.#getOptions(options), ); diff --git a/plugins/catalog-node/src/testUtils/catalogServiceMock.ts b/plugins/catalog-node/src/testUtils/catalogServiceMock.ts index 78962dc484..59c66c0b88 100644 --- a/plugins/catalog-node/src/testUtils/catalogServiceMock.ts +++ b/plugins/catalog-node/src/testUtils/catalogServiceMock.ts @@ -104,6 +104,5 @@ export namespace catalogServiceMock { validateEntity: jest.fn(), analyzeLocation: jest.fn(), streamEntities: jest.fn(), - streamEntityPages: jest.fn(), })); } diff --git a/plugins/catalog-node/src/testUtils/types.ts b/plugins/catalog-node/src/testUtils/types.ts index c8ada67dc5..a5bf125cfd 100644 --- a/plugins/catalog-node/src/testUtils/types.ts +++ b/plugins/catalog-node/src/testUtils/types.ts @@ -140,10 +140,5 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi { streamEntities( request?: StreamEntitiesRequest, options?: CatalogServiceRequestOptions | CatalogRequestOptions, - ): AsyncIterable; - - streamEntityPages( - request?: StreamEntitiesRequest, - options?: CatalogServiceRequestOptions | CatalogRequestOptions, ): AsyncIterable; }