diff --git a/.changeset/cuddly-flowers-film.md b/.changeset/cuddly-flowers-film.md new file mode 100644 index 0000000000..290bb74db6 --- /dev/null +++ b/.changeset/cuddly-flowers-film.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Avoid duplicate work by comparing previous processing rounds with the next diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 5f3bab63e7..794f2e6d0b 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -76,6 +76,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, + hash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', @@ -137,6 +138,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, + hash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 89bf3616ca..8e4a4ddc0c 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -162,10 +162,11 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { const hash = hashBuilder.digest('hex'); if (hash === previousHash) { - console.log('skipping ', entityRef); + // If nothing changed in our produced outputs, we cannot have any + // significant effect on our surroundings; therefore, we just abort + // without any updates / stitching. return; } - console.log('going ahead with ', entityRef); // If the result was marked as not OK, it signals that some part of the // processing pipeline threw an exception. This can happen both as part of diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 1511854a03..bb5f50c7a0 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -124,11 +124,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: UpdateProcessedEntityOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { id, errors } = options; + const { id, errors, hash } = options; await tx('refresh_state') .update({ errors, + hash, }) .where('entity_id', id); } @@ -494,7 +495,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { processedEntity: i.processed_entity ? (JSON.parse(i.processed_entity) as Entity) : undefined, - hash: i.hash, + hash: i.hash || '', nextUpdateAt: i.next_update_at, lastDiscoveryAt: i.last_discovery_at, state: i.cache