catalog-backend: use select for update skip locked for the work queue

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

.changeset/gorgeous-pumas-tickle.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Rely on `SELECT ... FOR UPDATE SKIP LOCKED` where available in order to speed up processing item acquisition and reduce work duplication.

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -355,8 +355,16 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
   ): Promise<GetProcessableEntitiesResult> {
     const tx = txOpaque as Knex.Transaction;
 
-    const items = await tx<DbRefreshStateRow>('refresh_state')
-      .select()
+    let itemsQuery = tx<DbRefreshStateRow>('refresh_state').select();
+
+    // This avoids duplication of work because of race conditions and is
+    // also fast because locked rows are ignored rather than blocking.
+    // It's only available in MySQL and PostgreSQL
+    if (['mysql', 'mysql2', 'pg'].includes(tx.client.config.client)) {
+      itemsQuery = itemsQuery.forUpdate().skipLocked();
+    }
+
+    const items = await itemsQuery
       .where('next_update_at', '<=', tx.fn.now())
       .limit(request.processBatchSize)
       .orderBy('next_update_at', 'asc');