From 18ab535c8368327d72626fc79de3ea55c18a10c4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 12 Jun 2021 18:11:21 +0200 Subject: [PATCH] catalog-backend: use select for update skip locked for the work queue Signed-off-by: Patrik Oldsberg --- .changeset/gorgeous-pumas-tickle.md | 5 +++++ .../src/next/database/DefaultProcessingDatabase.ts | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 .changeset/gorgeous-pumas-tickle.md diff --git a/.changeset/gorgeous-pumas-tickle.md b/.changeset/gorgeous-pumas-tickle.md new file mode 100644 index 0000000000..2f653fae84 --- /dev/null +++ b/.changeset/gorgeous-pumas-tickle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Rely on `SELECT ... FOR UPDATE SKIP LOCKED` where available in order to speed up processing item acquisition and reduce work duplication. diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 7eb2cd1a74..3569d4dcc6 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -355,8 +355,16 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ): Promise { const tx = txOpaque as Knex.Transaction; - const items = await tx('refresh_state') - .select() + let itemsQuery = tx('refresh_state').select(); + + // This avoids duplication of work because of race conditions and is + // also fast because locked rows are ignored rather than blocking. + // It's only available in MySQL and PostgreSQL + if (['mysql', 'mysql2', 'pg'].includes(tx.client.config.client)) { + itemsQuery = itemsQuery.forUpdate().skipLocked(); + } + + const items = await itemsQuery .where('next_update_at', '<=', tx.fn.now()) .limit(request.processBatchSize) .orderBy('next_update_at', 'asc');