diff --git a/.changeset/rare-peas-accept.md b/.changeset/rare-peas-accept.md new file mode 100644 index 0000000000..74597dda11 --- /dev/null +++ b/.changeset/rare-peas-accept.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Batch the writing of statuses after refreshes. This reduced the runtime on sqlite from 16s to 0.2s, and on pg from 60s to 1s on my machine, for the huge LDAP set. diff --git a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts index 9ce53d2850..91245c0789 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts @@ -70,7 +70,7 @@ export class DatabaseLocationsCatalog implements LocationsCatalog { async logUpdateSuccess( locationId: string, - entityName?: string, + entityName?: string | string[], ): Promise { await this.database.addLocationUpdateLogEvent( locationId, diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 52bb6e95ab..a3b2e0cb96 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -80,7 +80,10 @@ export type LocationsCatalog = { locations(): Promise; location(id: string): Promise; locationHistory(id: string): Promise; - logUpdateSuccess(locationId: string, entityName?: string): Promise; + logUpdateSuccess( + locationId: string, + entityName?: string | string[], + ): Promise; logUpdateFailure( locationId: string, error?: Error, diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index e38d3032b4..f4c3e75934 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -432,7 +432,7 @@ export class CommonDatabase implements Database { async addLocationUpdateLogEvent( locationId: string, status: DatabaseLocationUpdateLogStatus, - entityName?: string, + entityName?: string | string[], message?: string, ): Promise { // Remove log entries older than a day @@ -442,14 +442,20 @@ export class CommonDatabase implements Database { .where('created_at', '<', cutoff.toISOString()) .del(); - await this.database( - 'location_update_log', - ).insert({ - status, - location_id: locationId, - entity_name: entityName, - message, - }); + const items: Partial[] = [entityName] + .flat() + .map(n => ({ + status, + location_id: locationId, + entity_name: n, + message, + })); + + for (const chunk of lodash.chunk(items, BATCH_SIZE)) { + await this.database( + 'location_update_log', + ).insert(chunk); + } } private async updateEntitiesSearch( diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 1307b063f2..320ec1ef18 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -208,7 +208,7 @@ export type Database = { addLocationUpdateLogEvent( locationId: string, status: DatabaseLocationUpdateLogStatus, - entityName?: string, + entityName?: string | string[], message?: string, ): Promise; }; diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts index 40b48ca4ab..b9078df93f 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts @@ -372,10 +372,9 @@ describe('HigherOrderOperations', () => { '123', undefined, ); - expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith( - '123', + expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith('123', [ 'c1', - ); + ]); }); it('logs unsuccessful updates when reader fails', async () => { diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index 565c3671ee..b7e9c48fe5 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -188,12 +188,10 @@ export class HigherOrderOperations implements HigherOrderOperation { this.logger.info(`Posting update success markers`); - for (const entity of readerOutput.entities) { - await this.locationsCatalog.logUpdateSuccess( - location.id, - entity.entity.metadata.name, - ); - } + await this.locationsCatalog.logUpdateSuccess( + location.id, + readerOutput.entities.map(e => e.entity.metadata.name), + ); this.logger.info( `Wrote ${readerOutput.entities.length} entities from location ${