diff --git a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js index 7b0cf9b2a6..4c03405910 100644 --- a/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js @@ -83,10 +83,18 @@ exports.up = async function up(knex) { .inTable('refresh_state') .onDelete('CASCADE') .comment( - 'Entity ID which correspond to the ID in the refresh_state table', + 'Entity ID which corresponds to the ID in the refresh_state table', ); - table.text('etag').notNullable().comment('Etag to be used for caching'); - table.text('finalized_entity').notNullable().comment('The final entity'); + table + .text('hash') + .notNullable() + .comment( + 'Stable hash of the entity data, to be used for caching and avoiding redundant work', + ); + table + .text('final_entity') + .notNullable() + .comment('The JSON encoded final entity'); table.index('entity_id', 'final_entities_entity_id_idx'); }); diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index 5bddd47f41..e87bb01e1f 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -37,7 +37,7 @@ export class NextEntitiesCatalog implements EntitiesCatalog { 'final_entities', ).select(); - const entities = dbResponse.map(e => JSON.parse(e.finalized_entity)); + const entities = dbResponse.map(e => JSON.parse(e.final_entity)); return { entities, diff --git a/plugins/catalog-backend/src/next/Stitcher.test.ts b/plugins/catalog-backend/src/next/Stitcher.test.ts index 327d922c07..f38af384fa 100644 --- a/plugins/catalog-backend/src/next/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/Stitcher.test.ts @@ -74,12 +74,12 @@ describe('Stitcher', () => { await stitcher.stitch(new Set(['k:ns/n'])); - let firstEtag: string; + let firstHash: string; await db.transaction(async tx => { const entities = await tx('final_entities'); expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].finalized_entity); + const entity = JSON.parse(entities[0].final_entity); expect(entity).toEqual({ relations: [ { @@ -105,12 +105,13 @@ describe('Stitcher', () => { }, }); - firstEtag = entity.metadata.etag; + expect(entity.metadata.etag).toEqual(entities[0].hash); + firstHash = entities[0].hash; const search = await tx('search'); expect(search).toEqual( expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' }, + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, { entity_id: 'my-id', key: 'apiversion', value: 'a' }, { entity_id: 'my-id', key: 'kind', value: 'k' }, { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, @@ -127,9 +128,9 @@ describe('Stitcher', () => { await db.transaction(async tx => { const entities = await tx('final_entities'); expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].finalized_entity); - expect(entities[0].etag).toEqual(firstEtag); - expect(entity.metadata.etag).toEqual(firstEtag); + const entity = JSON.parse(entities[0].final_entity); + expect(entities[0].hash).toEqual(firstHash); + expect(entity.metadata.etag).toEqual(firstHash); }); // Now add one more relation and re-stitch @@ -150,7 +151,7 @@ describe('Stitcher', () => { const entities = await tx('final_entities'); expect(entities.length).toBe(1); - const entity = JSON.parse(entities[0].finalized_entity); + const entity = JSON.parse(entities[0].final_entity); expect(entity).toEqual({ relations: expect.arrayContaining([ { @@ -184,14 +185,14 @@ describe('Stitcher', () => { }, }); - expect(entities[0].etag).not.toEqual(firstEtag); - expect(entities[0].etag).toEqual(entity.metadata.etag); + expect(entities[0].hash).not.toEqual(firstHash); + expect(entities[0].hash).toEqual(entity.metadata.etag); const search = await tx('search'); expect(search).toEqual( expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' }, - { entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/third' }, + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, { entity_id: 'my-id', key: 'apiversion', value: 'a' }, { entity_id: 'my-id', key: 'kind', value: 'k' }, { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, diff --git a/plugins/catalog-backend/src/next/Stitcher.ts b/plugins/catalog-backend/src/next/Stitcher.ts index cd9220e934..bdc028d923 100644 --- a/plugins/catalog-backend/src/next/Stitcher.ts +++ b/plugins/catalog-backend/src/next/Stitcher.ts @@ -31,11 +31,11 @@ const BATCH_SIZE = 50; export type DbFinalEntitiesRow = { entity_id: string; - etag: string; - finalized_entity: string; + hash: string; + final_entity: string; }; -function generateEntityEtag(entity: Entity) { +function generateStableHash(entity: Entity) { return createHash('sha1') .update(stableStringify({ ...entity })) .digest('hex'); @@ -64,7 +64,7 @@ export class Stitcher { processedEntity?: string; errors: string; incomingReferenceCount: string | number; - previousEtag?: string; + previousHash?: string; relationType?: string; relationTarget?: string; }> = await tx @@ -79,7 +79,7 @@ export class Stitcher { processedEntity: 'refresh_state.processed_entity', errors: 'refresh_state.errors', incomingReferenceCount: 'incoming_references.count', - previousEtag: 'final_entities.etag', + previousHash: 'final_entities.hash', relationType: 'relations.type', relationTarget: 'relations.target_entity_ref', }) @@ -111,7 +111,7 @@ export class Stitcher { processedEntity, // errors, incomingReferenceCount, - previousEtag, + previousHash, } = result[0]; // If there was no processed entity in place, the target hasn't been @@ -125,7 +125,8 @@ export class Stitcher { return; } - // Grab the processed entity and stitch all of the relevant data into it + // Grab the processed entity and stitch all of the relevant data into + // it const entity = JSON.parse(processedEntity) as Entity; const isOrphan = Number(incomingReferenceCount) === 0; @@ -137,33 +138,38 @@ export class Stitcher { }; } - // TODO: entityRef is lower case and should be uppercase in the final result + // TODO: entityRef is lower case and should be uppercase in the final + // result entity.relations = result - .filter(row => row.relationType) + .filter(row => row.relationType /* exclude null row, if relevant */) .map(row => ({ type: row.relationType!, target: parseEntityRef(row.relationTarget!), })); // If the output entity was actually not changed, just abort - const etag = generateEntityEtag(entity); - if (etag === previousEtag) { + const hash = generateStableHash(entity); + if (hash === previousHash) { this.logger.debug(`Skipped stitching of ${entityRef}, no changes`); return; } entity.metadata.uid = entityId; entity.metadata.generation = 1; - entity.metadata.etag = etag; + if (!entity.metadata.etag) { + // If the original data source did not have its own etag handling, + // use the hash as a good-quality etag + entity.metadata.etag = hash; + } await tx('final_entities') .insert({ entity_id: entityId, - finalized_entity: JSON.stringify(entity), - etag, + final_entity: JSON.stringify(entity), + hash, }) .onConflict('entity_id') - .merge(['finalized_entity', 'etag']); + .merge(['final_entity', 'hash']); const searchEntries = buildEntitySearch(entityId, entity); await tx('search').where({ entity_id: entityId }).delete(); diff --git a/plugins/catalog-backend/src/next/search.test.ts b/plugins/catalog-backend/src/next/search.test.ts index be9a98fe69..ff1cba01b9 100644 --- a/plugins/catalog-backend/src/next/search.test.ts +++ b/plugins/catalog-backend/src/next/search.test.ts @@ -152,8 +152,8 @@ describe('search', () => { key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE, }, - { entity_id: 'eid', key: 'relations', value: 't1:k:ns/a' }, - { entity_id: 'eid', key: 'relations', value: 't2:k:ns/b' }, + { entity_id: 'eid', key: 'relations.t1', value: 'k:ns/a' }, + { entity_id: 'eid', key: 'relations.t2', value: 'k:ns/b' }, ]); }); }); diff --git a/plugins/catalog-backend/src/next/search.ts b/plugins/catalog-backend/src/next/search.ts index 4aa8bae1a8..5ff0721f91 100644 --- a/plugins/catalog-backend/src/next/search.ts +++ b/plugins/catalog-backend/src/next/search.ts @@ -182,8 +182,8 @@ export function buildEntitySearch( // Visit relations for (const relation of entity.relations ?? []) { raw.push({ - key: 'relations', - value: `${relation.type}:${stringifyEntityRef(relation.target)}`, + key: `relations.${relation.type}`, + value: stringifyEntityRef(relation.target), }); }