paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

fix(catalog): remove streamEntityPages function

Browse Source 
and make the streamEntities work as the pages function did earlier

Signed-off-by: Hellgren Heikki <heikki.hellgren@op.fi>
This commit is contained in:
Hellgren Heikki
2025-09-03 08:48:10 +03:00
parent a87ab00712
commit 9118c56520
13 changed files with 8 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/olive-moons-burn.md
+5 -16
View File
@@ -4,7 +4,7 @@
'@backstage/plugin-catalog-node': minor
---
Introduced new `streamEntities` and `streamEntityPages` async generator methods for the catalog.
Introduced new `streamEntities` async generator method for the catalog.
Catalog API and Catalog Service now includes a `streamEntities` method that allows for streaming entities from the catalog.
This method is designed to handle large datasets efficiently by processing entities in a stream rather than loading them
@@ -14,22 +14,11 @@ or fetch all entities at once.
Example usage:
```ts
const stream = catalogClient.streamEntities({}, { token });
for await (const entity of stream) {
// Handle entity
}
```
Additionally, a `streamEntityPages` method is available that streams entities in pages, allowing for batch processing of entities.
This method is more efficient than `streamEntities` when you can process entities in chunks.
Example usage:
```ts
const pageStream = catalogClient.streamEntityPages(
{ pageSize: 100 },
{ token },
);
const pageStream = catalogClient.streamEntities({ pageSize: 100 }, { token });
for await (const page of pageStream) {
// Handle page of entities
for (const entity of page) {
console.log(entity);
}
}
```
packages/catalog-client/report-testUtils.api.md
+1 -3
View File
@@ -71,9 +71,7 @@ export class InMemoryCatalogClient implements CatalogApi {
// (undocumented)
removeLocationById(_id: string): Promise<void>;
// (undocumented)
streamEntities(request?: StreamEntitiesRequest): AsyncIterable<Entity>;
// (undocumented)
streamEntityPages(request?: StreamEntitiesRequest): AsyncIterable<Entity[]>;
streamEntities(request?: StreamEntitiesRequest): AsyncIterable<Entity[]>;
// (undocumented)
validateEntity(
_entity: Entity,
packages/catalog-client/report.api.md
-8
View File
@@ -91,10 +91,6 @@ export interface CatalogApi {
streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity>;
streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity[]>;
validateEntity(
entity: Entity,
@@ -181,10 +177,6 @@ export class CatalogClient implements CatalogApi {
streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity>;
streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity[]>;
validateEntity(
entity: Entity,
packages/catalog-client/src/CatalogClient.test.ts
-9
View File
@@ -581,15 +581,6 @@ describe('CatalogClient', () => {
it('should stream entities', async () => {
const stream = client.streamEntities({}, { token });
const results: Entity[] = [];
for await (const entity of stream) {
results.push(entity);
}
expect(results).toEqual(defaultResponse.items);
});
it('should stream entity pages', async () => {
const stream = client.streamEntityPages({}, { token });
const results: Entity[][] = [];
for await (const entityPage of stream) {
results.push(entityPage);
packages/catalog-client/src/CatalogClient.ts
-15
View File
@@ -464,21 +464,6 @@ export class CatalogClient implements CatalogApi {
async *streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity> {
const pages = this.streamEntityPages(request, options);
for await (const page of pages) {
for (const entity of page) {
yield entity;
}
}
}
/**
* {@inheritdoc CatalogApi.streamEntityPages}
*/
async *streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity[]> {
let cursor: string | undefined = undefined;
const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT;
packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts
-10
View File
@@ -126,16 +126,6 @@ describe('InMemoryCatalogClient', () => {
it('streamEntities', async () => {
const client = new InMemoryCatalogClient({ entities });
const stream = client.streamEntities();
const results: Entity[] = [];
for await (const entity of stream) {
results.push(entity);
}
expect(results).toEqual(entities);
});
it('streamEntityPages', async () => {
const client = new InMemoryCatalogClient({ entities });
const stream = client.streamEntityPages();
const results: Entity[][] = [];
for await (const page of stream) {
results.push(page);
packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts
-11
View File
@@ -283,17 +283,6 @@ export class InMemoryCatalogClient implements CatalogApi {
async *streamEntities(
request?: StreamEntitiesRequest,
): AsyncIterable<Entity> {
const pages = this.streamEntityPages(request);
for await (const page of pages) {
for (const entity of page) {
yield entity;
}
}
}
async *streamEntityPages(
request?: StreamEntitiesRequest,
): AsyncIterable<Entity[]> {
let cursor: string | undefined = undefined;
const limit = request?.pageSize ?? DEFAULT_STREAM_ENTITIES_LIMIT;
packages/catalog-client/src/types/api.ts
+1 -15
View File
@@ -710,7 +710,7 @@ export interface CatalogApi {
/**
* Asynchronously streams entities from the catalog. Uses `queryEntities`
* to fetch entities in batches, and yields them one by one.
* to fetch entities in batches, and yields them one page at a time.
*
* @public
*
@@ -720,19 +720,5 @@ export interface CatalogApi {
streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity>;
/**
* Asynchronously streams entity pages from the catalog. Uses `queryEntities`
* to fetch entities in batches, and yields them one page at a time.
*
* @public
*
* @param request - Request parameters
* @param options - Additional options
*/
streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogRequestOptions,
): AsyncIterable<Entity[]>;
}
plugins/catalog-node/report-testUtils.api.md
-5
View File
@@ -111,11 +111,6 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi {
streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogServiceRequestOptions | CatalogRequestOptions,
): AsyncIterable<Entity>;
// (undocumented)
streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogServiceRequestOptions | CatalogRequestOptions,
): AsyncIterable<Entity[]>;
// (undocumented)
validateEntity(
plugins/catalog-node/report.api.md
-5
View File
@@ -199,11 +199,6 @@ export interface CatalogService {
streamEntities(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity>;
// (undocumented)
streamEntityPages(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity[]>;
// (undocumented)
validateEntity(
plugins/catalog-node/src/catalogService.ts
+1 -16
View File
@@ -146,11 +146,6 @@ export interface CatalogService {
streamEntities(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity>;
streamEntityPages(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity[]>;
}
@@ -334,18 +329,8 @@ class DefaultCatalogService implements CatalogService {
async *streamEntities(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity> {
yield* this.#catalogApi.streamEntities(
request,
await this.#getOptions(options),
);
}
async *streamEntityPages(
request: StreamEntitiesRequest | undefined,
options: CatalogServiceRequestOptions,
): AsyncIterable<Entity[]> {
yield* this.#catalogApi.streamEntityPages(
yield* this.#catalogApi.streamEntities(
request,
await this.#getOptions(options),
);
plugins/catalog-node/src/testUtils/catalogServiceMock.ts
-1
View File
@@ -104,6 +104,5 @@ export namespace catalogServiceMock {
validateEntity: jest.fn(),
analyzeLocation: jest.fn(),
streamEntities: jest.fn(),
streamEntityPages: jest.fn(),
}));
}
plugins/catalog-node/src/testUtils/types.ts
-5
View File
@@ -140,10 +140,5 @@ export interface CatalogServiceMock extends CatalogService, CatalogApi {
streamEntities(
request?: StreamEntitiesRequest,
options?: CatalogServiceRequestOptions | CatalogRequestOptions,
): AsyncIterable<Entity>;
streamEntityPages(
request?: StreamEntitiesRequest,
options?: CatalogServiceRequestOptions | CatalogRequestOptions,
): AsyncIterable<Entity[]>;
}