From d373bae7aae26b30af620447a5e9c49301c3ed3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Aug 2021 16:35:23 +0200 Subject: [PATCH 1/4] add the hash column MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/knexfile.js | 26 ++++++++++++ .../20210813143113_add_refresh_state_hash.js | 40 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 plugins/catalog-backend/knexfile.js create mode 100644 plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js diff --git a/plugins/catalog-backend/knexfile.js b/plugins/catalog-backend/knexfile.js new file mode 100644 index 0000000000..4c8be42673 --- /dev/null +++ b/plugins/catalog-backend/knexfile.js @@ -0,0 +1,26 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file makes it possible to run "yarn knex migrate:make some_file_name" +// to assist in making new migrations +module.exports = { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + migrations: { + directory: './migrations', + }, +}; diff --git a/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js b/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js new file mode 100644 index 0000000000..5ac9a95689 --- /dev/null +++ b/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js @@ -0,0 +1,40 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('refresh_state', table => { + table + .text('hash') + .nullable() + .comment( + 'A hash of the processed contents, used to avoid duplicate work', + ); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('refresh_state', table => { + table.dropColumn('hash'); + }); +}; From e671ac95920875c414817d6e709ace9b47734d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Aug 2021 19:02:15 +0200 Subject: [PATCH 2/4] compute hash and update on each round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../next/DefaultCatalogProcessingEngine.ts | 29 ++++++++++++++++++- .../DefaultProcessingDatabase.test.ts | 11 ++++++- .../database/DefaultProcessingDatabase.ts | 3 ++ .../src/next/database/tables.ts | 1 + .../src/next/database/types.ts | 3 ++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 0ae20c94b1..89bf3616ca 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -20,6 +20,8 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { serializeError } from '@backstage/errors'; +import { createHash } from 'crypto'; +import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; import { ProcessingDatabase, RefreshStateItem } from './database/types'; import { CatalogProcessingOrchestrator } from './processing/types'; @@ -125,7 +127,14 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { }, processTask: async item => { try { - const { id, state, unprocessedEntity, entityRef, locationKey } = item; + const { + id, + state, + unprocessedEntity, + entityRef, + locationKey, + hash: previousHash, + } = item; const result = await this.orchestrator.process({ entity: unprocessedEntity, state, @@ -142,6 +151,22 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { result.errors.map(e => serializeError(e)), ); + let hashBuilder = createHash('sha1').update(errorsString); + if (result.ok) { + hashBuilder = hashBuilder + .update(stableStringify({ ...result.completedEntity })) + .update(stableStringify([...result.deferredEntities])) + .update(stableStringify([...result.relations])) + .update(stableStringify({ ...result.state })); + } + + const hash = hashBuilder.digest('hex'); + if (hash === previousHash) { + console.log('skipping ', entityRef); + return; + } + console.log('going ahead with ', entityRef); + // If the result was marked as not OK, it signals that some part of the // processing pipeline threw an exception. This can happen both as part of // non-catastrophic things such as due to validation errors, as well as if @@ -154,6 +179,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { await this.processingDatabase.updateProcessedEntityErrors(tx, { id, errors: errorsString, + hash, }); }); await this.stitcher.stitch( @@ -167,6 +193,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { await this.processingDatabase.updateProcessedEntity(tx, { id, processedEntity: result.completedEntity, + hash, state: result.state, errors: errorsString, relations: result.relations, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 6a0626afc4..8b59e5b5da 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -206,6 +206,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, + hash: '', state: new Map(), relations: [], deferredEntities: [], @@ -224,6 +225,7 @@ describe('Default Processing Database', () => { const options = { id, processedEntity, + hash: '', state: new Map(), relations: [], deferredEntities: [], @@ -250,7 +252,11 @@ describe('Default Processing Database', () => { await db.transaction(tx => expect( - db.updateProcessedEntity(tx, { ...options, locationKey: 'fail' }), + db.updateProcessedEntity(tx, { + ...options, + hash: '', + locationKey: 'fail', + }), ).rejects.toThrow( `Conflicting write of processing result for ${id} with location key 'fail'`, ), @@ -280,6 +286,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, + hash: '', state, relations: [], deferredEntities: [], @@ -336,6 +343,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, + hash: '', state: new Map(), relations: relations, deferredEntities: [], @@ -387,6 +395,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, + hash: '', state: new Map(), relations: [], deferredEntities, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 1f1ea4bf84..1511854a03 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -60,6 +60,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { id, processedEntity, + hash, state, errors, relations, @@ -69,6 +70,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), + hash, cache: JSON.stringify(state), errors, location_key: locationKey, @@ -492,6 +494,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { processedEntity: i.processed_entity ? (JSON.parse(i.processed_entity) as Entity) : undefined, + hash: i.hash, nextUpdateAt: i.next_update_at, lastDiscoveryAt: i.last_discovery_at, state: i.cache diff --git a/plugins/catalog-backend/src/next/database/tables.ts b/plugins/catalog-backend/src/next/database/tables.ts index 395694131d..594b7a3ade 100644 --- a/plugins/catalog-backend/src/next/database/tables.ts +++ b/plugins/catalog-backend/src/next/database/tables.ts @@ -25,6 +25,7 @@ export type DbRefreshStateRow = { entity_ref: string; unprocessed_entity: string; processed_entity?: string; + hash?: string; cache?: string; next_update_at: string; last_discovery_at: string; // remove? diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index 78199900e4..c559895d27 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -34,6 +34,7 @@ export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { id: string; processedEntity: Entity; + hash: string; state?: Map; errors?: string; relations: EntityRelationSpec[]; @@ -44,6 +45,7 @@ export type UpdateProcessedEntityOptions = { export type UpdateProcessedEntityErrorsOptions = { id: string; errors?: string; + hash: string; }; export type RefreshStateItem = { @@ -51,6 +53,7 @@ export type RefreshStateItem = { entityRef: string; unprocessedEntity: Entity; processedEntity?: Entity; + hash: string; nextUpdateAt: string; lastDiscoveryAt: string; // remove? state: Map; From 61aa6526f70c67266189d90fa45310cc88336cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Aug 2021 19:03:13 +0200 Subject: [PATCH 3/4] add changeset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/cuddly-flowers-film.md | 5 +++++ .../src/next/DefaultCatalogProcessingEngine.test.ts | 2 ++ .../src/next/DefaultCatalogProcessingEngine.ts | 5 +++-- .../src/next/database/DefaultProcessingDatabase.ts | 5 +++-- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 .changeset/cuddly-flowers-film.md diff --git a/.changeset/cuddly-flowers-film.md b/.changeset/cuddly-flowers-film.md new file mode 100644 index 0000000000..290bb74db6 --- /dev/null +++ b/.changeset/cuddly-flowers-film.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Avoid duplicate work by comparing previous processing rounds with the next diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 5f3bab63e7..794f2e6d0b 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -76,6 +76,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, + hash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', @@ -137,6 +138,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, + hash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 89bf3616ca..8e4a4ddc0c 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -162,10 +162,11 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { const hash = hashBuilder.digest('hex'); if (hash === previousHash) { - console.log('skipping ', entityRef); + // If nothing changed in our produced outputs, we cannot have any + // significant effect on our surroundings; therefore, we just abort + // without any updates / stitching. return; } - console.log('going ahead with ', entityRef); // If the result was marked as not OK, it signals that some part of the // processing pipeline threw an exception. This can happen both as part of diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 1511854a03..bb5f50c7a0 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -124,11 +124,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: UpdateProcessedEntityOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { id, errors } = options; + const { id, errors, hash } = options; await tx('refresh_state') .update({ errors, + hash, }) .where('entity_id', id); } @@ -494,7 +495,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { processedEntity: i.processed_entity ? (JSON.parse(i.processed_entity) as Entity) : undefined, - hash: i.hash, + hash: i.hash || '', nextUpdateAt: i.next_update_at, lastDiscoveryAt: i.last_discovery_at, state: i.cache From 73f3216bc0c239f8b84a5909bdd2ce2ab3675346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Aug 2021 21:53:23 +0200 Subject: [PATCH 4/4] add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../20210813143113_add_refresh_state_hash.js | 4 +- .../DefaultCatalogProcessingEngine.test.ts | 78 ++++++++++++++++++- .../next/DefaultCatalogProcessingEngine.ts | 17 ++-- .../src/next/NextCatalogBuilder.ts | 2 + .../DefaultProcessingDatabase.test.ts | 18 +++-- .../database/DefaultProcessingDatabase.ts | 12 +-- .../src/next/database/tables.ts | 2 +- .../src/next/database/types.ts | 6 +- 8 files changed, 109 insertions(+), 30 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js b/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js index 5ac9a95689..29a102e876 100644 --- a/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js +++ b/plugins/catalog-backend/migrations/20210813143113_add_refresh_state_hash.js @@ -22,7 +22,7 @@ exports.up = async function up(knex) { await knex.schema.alterTable('refresh_state', table => { table - .text('hash') + .text('result_hash') .nullable() .comment( 'A hash of the processed contents, used to avoid duplicate work', @@ -35,6 +35,6 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { await knex.schema.alterTable('refresh_state', table => { - table.dropColumn('hash'); + table.dropColumn('result_hash'); }); }; diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 794f2e6d0b..898ef08f6c 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { Hash } from 'crypto'; import waitForExpect from 'wait-for-expect'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; @@ -33,6 +34,10 @@ describe('DefaultCatalogProcessingEngine', () => { const stitcher = { stitch: jest.fn(), } as unknown as jest.Mocked; + const hash = { + update: () => hash, + digest: jest.fn(), + } as unknown as jest.Mocked; beforeEach(() => { jest.resetAllMocks(); @@ -57,6 +62,7 @@ describe('DefaultCatalogProcessingEngine', () => { db, orchestrator, stitcher, + () => hash, ); db.transaction.mockImplementation(cb => cb((() => {}) as any)); @@ -76,7 +82,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, - hash: '', + resultHash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', @@ -118,6 +124,7 @@ describe('DefaultCatalogProcessingEngine', () => { db, orchestrator, stitcher, + () => hash, ); db.transaction.mockImplementation(cb => cb((() => {}) as any)); @@ -138,7 +145,7 @@ describe('DefaultCatalogProcessingEngine', () => { kind: 'Location', metadata: { name: 'test' }, }, - hash: '', + resultHash: '', state: new Map(), nextUpdateAt: '', lastDiscoveryAt: '', @@ -160,4 +167,71 @@ describe('DefaultCatalogProcessingEngine', () => { }); await engine.stop(); }); + + it('runs fully when hash mismatches, early-outs when hash matches', async () => { + const entity = { + apiVersion: '1', + kind: 'Location', + metadata: { name: 'test' }, + }; + + const refreshState = { + id: '', + entityRef: '', + unprocessedEntity: entity, + resultHash: 'the matching hash', + state: new Map(), + nextUpdateAt: '', + lastDiscoveryAt: '', + }; + + hash.digest.mockReturnValue('the matching hash'); + + orchestrator.process.mockResolvedValue({ + ok: true, + completedEntity: entity, + relations: [], + errors: [], + deferredEntities: [], + state: new Map(), + }); + + const engine = new DefaultCatalogProcessingEngine( + getVoidLogger(), + [], + db, + orchestrator, + stitcher, + () => hash, + ); + + db.transaction.mockImplementation(cb => cb((() => {}) as any)); + + db.getProcessableEntities + .mockResolvedValueOnce({ + items: [{ ...refreshState, resultHash: 'NOT RIGHT' }], + }) + .mockResolvedValue({ items: [] }); + + await engine.start(); + + await waitForExpect(() => { + expect(orchestrator.process).toBeCalledTimes(1); + expect(hash.digest).toBeCalledTimes(1); + expect(db.updateProcessedEntity).toBeCalledTimes(1); + }); + + db.getProcessableEntities + .mockReset() + .mockResolvedValueOnce({ items: [refreshState] }) + .mockResolvedValue({ items: [] }); + + await waitForExpect(() => { + expect(orchestrator.process).toBeCalledTimes(2); + expect(hash.digest).toBeCalledTimes(2); + expect(db.updateProcessedEntity).toBeCalledTimes(1); + }); + + await engine.stop(); + }); }); diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 8e4a4ddc0c..6468fa5187 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -20,7 +20,7 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { serializeError } from '@backstage/errors'; -import { createHash } from 'crypto'; +import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; import { ProcessingDatabase, RefreshStateItem } from './database/types'; @@ -91,6 +91,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { private readonly processingDatabase: ProcessingDatabase, private readonly orchestrator: CatalogProcessingOrchestrator, private readonly stitcher: Stitcher, + private readonly createHash: () => Hash, ) {} async start() { @@ -133,7 +134,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { unprocessedEntity, entityRef, locationKey, - hash: previousHash, + resultHash: previousResultHash, } = item; const result = await this.orchestrator.process({ entity: unprocessedEntity, @@ -151,17 +152,17 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { result.errors.map(e => serializeError(e)), ); - let hashBuilder = createHash('sha1').update(errorsString); + let hashBuilder = this.createHash().update(errorsString); if (result.ok) { hashBuilder = hashBuilder .update(stableStringify({ ...result.completedEntity })) .update(stableStringify([...result.deferredEntities])) .update(stableStringify([...result.relations])) - .update(stableStringify({ ...result.state })); + .update(stableStringify(Object.fromEntries(result.state))); } - const hash = hashBuilder.digest('hex'); - if (hash === previousHash) { + const resultHash = hashBuilder.digest('hex'); + if (resultHash === previousResultHash) { // If nothing changed in our produced outputs, we cannot have any // significant effect on our surroundings; therefore, we just abort // without any updates / stitching. @@ -180,7 +181,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { await this.processingDatabase.updateProcessedEntityErrors(tx, { id, errors: errorsString, - hash, + resultHash, }); }); await this.stitcher.stitch( @@ -194,7 +195,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { await this.processingDatabase.updateProcessedEntity(tx, { id, processedEntity: result.completedEntity, - hash, + resultHash, state: result.state, errors: errorsString, relations: result.relations, diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index a525dacd51..fb12077508 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -31,6 +31,7 @@ import { } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; +import { createHash } from 'crypto'; import lodash from 'lodash'; import { Logger } from 'winston'; import { @@ -310,6 +311,7 @@ export class NextCatalogBuilder { processingDatabase, orchestrator, stitcher, + () => createHash('sha1'), ); const locationsCatalog = new DatabaseLocationsCatalog(db); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 8b59e5b5da..5103c25dbc 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -19,8 +19,8 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Knex } from 'knex'; -import { Logger } from 'winston'; import * as uuid from 'uuid'; +import { Logger } from 'winston'; import { DatabaseManager } from './DatabaseManager'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { @@ -206,7 +206,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, - hash: '', + resultHash: '', state: new Map(), relations: [], deferredEntities: [], @@ -225,7 +225,7 @@ describe('Default Processing Database', () => { const options = { id, processedEntity, - hash: '', + resultHash: '', state: new Map(), relations: [], deferredEntities: [], @@ -254,7 +254,7 @@ describe('Default Processing Database', () => { expect( db.updateProcessedEntity(tx, { ...options, - hash: '', + resultHash: '', locationKey: 'fail', }), ).rejects.toThrow( @@ -286,7 +286,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, - hash: '', + resultHash: '', state, relations: [], deferredEntities: [], @@ -302,7 +302,9 @@ describe('Default Processing Database', () => { expect(entities[0].processed_entity).toEqual( JSON.stringify(processedEntity), ); - expect(entities[0].cache).toEqual(JSON.stringify(state)); + expect(entities[0].cache).toEqual( + JSON.stringify(Object.fromEntries(state)), + ); expect(entities[0].errors).toEqual("['something broke']"); expect(entities[0].location_key).toEqual('key'); }, @@ -343,7 +345,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, - hash: '', + resultHash: '', state: new Map(), relations: relations, deferredEntities: [], @@ -395,7 +397,7 @@ describe('Default Processing Database', () => { db.updateProcessedEntity(tx, { id, processedEntity, - hash: '', + resultHash: '', state: new Map(), relations: [], deferredEntities, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index bb5f50c7a0..177bf979ab 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -60,7 +60,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const { id, processedEntity, - hash, + resultHash, state, errors, relations, @@ -70,8 +70,8 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), - hash, - cache: JSON.stringify(state), + result_hash: resultHash, + cache: JSON.stringify(Object.fromEntries(state || [])), errors, location_key: locationKey, }) @@ -124,12 +124,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { options: UpdateProcessedEntityOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { id, errors, hash } = options; + const { id, errors, resultHash } = options; await tx('refresh_state') .update({ errors, - hash, + result_hash: resultHash, }) .where('entity_id', id); } @@ -495,7 +495,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { processedEntity: i.processed_entity ? (JSON.parse(i.processed_entity) as Entity) : undefined, - hash: i.hash || '', + resultHash: i.result_hash || '', nextUpdateAt: i.next_update_at, lastDiscoveryAt: i.last_discovery_at, state: i.cache diff --git a/plugins/catalog-backend/src/next/database/tables.ts b/plugins/catalog-backend/src/next/database/tables.ts index 594b7a3ade..d735607554 100644 --- a/plugins/catalog-backend/src/next/database/tables.ts +++ b/plugins/catalog-backend/src/next/database/tables.ts @@ -25,7 +25,7 @@ export type DbRefreshStateRow = { entity_ref: string; unprocessed_entity: string; processed_entity?: string; - hash?: string; + result_hash?: string; cache?: string; next_update_at: string; last_discovery_at: string; // remove? diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index c559895d27..c93b90084d 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -34,7 +34,7 @@ export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { id: string; processedEntity: Entity; - hash: string; + resultHash: string; state?: Map; errors?: string; relations: EntityRelationSpec[]; @@ -45,7 +45,7 @@ export type UpdateProcessedEntityOptions = { export type UpdateProcessedEntityErrorsOptions = { id: string; errors?: string; - hash: string; + resultHash: string; }; export type RefreshStateItem = { @@ -53,7 +53,7 @@ export type RefreshStateItem = { entityRef: string; unprocessedEntity: Entity; processedEntity?: Entity; - hash: string; + resultHash: string; nextUpdateAt: string; lastDiscoveryAt: string; // remove? state: Map;