From 375b546fa1c48c4d8882f90eabbc0e0320329936 Mon Sep 17 00:00:00 2001 From: Michael Walsh Date: Fri, 20 Mar 2026 17:19:09 +0100 Subject: [PATCH] fix(catalog): Fix catalog refresh_state deadlock when running multiple replicas getProcessableEntities uses SELECT ... FOR UPDATE SKIP LOCKED to prevent concurrent processors from selecting the same rows, but it was called with a raw Knex instance instead of a transaxction. This meant the row locks were released immediately after the SELECT, before the subsequent UPDATE executed - making SKIP LOCKED ineffective and allowing multiple replicas to update overlapping rows, causing PostgreSQL deadlock (error 40P01). Wrapping the call in a transaction ensures the locks are held through the UPDATE, so concurrent replicas correctly skip already-claimed rows. Signed-off-by: Michael Walsh --- .changeset/fix-catalog-refresh-state-deadlock.md | 5 +++++ .../src/processing/DefaultCatalogProcessingEngine.ts | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .changeset/fix-catalog-refresh-state-deadlock.md diff --git a/.changeset/fix-catalog-refresh-state-deadlock.md b/.changeset/fix-catalog-refresh-state-deadlock.md new file mode 100644 index 0000000000..867bafce49 --- /dev/null +++ b/.changeset/fix-catalog-refresh-state-deadlock.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed a deadlock in the catalog processing loop that occurred when running multiple replicas. The `getProcessableEntities` method used `SELECT ... FOR UPDATE SKIP LOCKED` to prevent concurrent processors from claiming the same rows, but the call was not wrapped in a transaction, so the row locks were released before the subsequent `UPDATE` executed. This allowed multiple replicas to select and update overlapping rows, causing PostgreSQL deadlock errors (code 40P01). diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 9416fceea1..0f3f764901 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -143,8 +143,10 @@ export class DefaultCatalogProcessingEngine { loadTasks: async count => { try { const { items } = - await this.processingDatabase.getProcessableEntities(this.knex, { - processBatchSize: count, + await this.processingDatabase.transaction(async tx => { + return this.processingDatabase.getProcessableEntities(tx, { + processBatchSize: count, + }); }); return items; } catch (error) {