From 4a22d3786d801a57302de45fcfcfdb3a918119a3 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 24 Oct 2024 16:53:32 +0200 Subject: [PATCH] chore: remove the refresh_state table from the read path, and have `entity_ref` in `final_entities` Signed-off-by: blam Signed-off-by: blam --- ...104700_add_entity_ref_to_final_entities.js | 47 ++++++++++++ .../operations/stitcher/performStitching.ts | 3 +- .../catalog-backend/src/database/tables.ts | 1 + .../service/DefaultEntitiesCatalog.test.ts | 1 + .../src/service/DefaultEntitiesCatalog.ts | 36 +++------ .../src/tests/migrations.test.ts | 75 +++++++++++++++++++ 6 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 plugins/catalog-backend/migrations/20241024104700_add_entity_ref_to_final_entities.js diff --git a/plugins/catalog-backend/migrations/20241024104700_add_entity_ref_to_final_entities.js b/plugins/catalog-backend/migrations/20241024104700_add_entity_ref_to_final_entities.js new file mode 100644 index 0000000000..4ae97d810c --- /dev/null +++ b/plugins/catalog-backend/migrations/20241024104700_add_entity_ref_to_final_entities.js @@ -0,0 +1,47 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('final_entities', table => { + table + .text('entity_ref') + .nullable() + .comment( + 'The entity reference of the entity that was created from the catalog processing', + ); + }); + + // Set the default entity_ref in final_entities from the refresh_state table + await knex.raw( + `UPDATE final_entities SET entity_ref = refresh_state.entity_ref FROM refresh_state WHERE refresh_state.entity_id = final_entities.entity_id`, + ); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('final_entities', table => { + table.dropColumn('entity_ref'); + }); +}; diff --git a/plugins/catalog-backend/src/database/operations/stitcher/performStitching.ts b/plugins/catalog-backend/src/database/operations/stitcher/performStitching.ts index 0832ba5db8..dc990fefc3 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/performStitching.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/performStitching.ts @@ -212,11 +212,12 @@ export async function performStitching(options: { final_entity: JSON.stringify(entity), hash, last_updated_at: knex.fn.now(), + entity_ref: entityRef, }) .where('entity_id', entityId) .where('stitch_ticket', stitchTicket) .onConflict('entity_id') - .merge(['final_entity', 'hash', 'last_updated_at']); + .merge(['final_entity', 'hash', 'last_updated_at', 'entity_ref']); const markDeferred = async () => { if (options.strategy.mode !== 'deferred') { diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 57731e3d27..df171ebb18 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -179,6 +179,7 @@ export type DbFinalEntitiesRow = { stitch_ticket: string; final_entity?: string; last_updated_at?: string | Date; + entity_ref?: string; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 92acd9e3ce..5b369bfa41 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -76,6 +76,7 @@ describe('DefaultEntitiesCatalog', () => { await knex('final_entities').insert({ entity_id: id, + entity_ref: entityRef, final_entity: entityJson, hash: 'h', stitch_ticket: '', diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index c71f242969..830abd16ab 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -251,13 +251,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }); if (!request?.order) { - entitiesQuery = entitiesQuery - .leftOuterJoin( - 'refresh_state', - 'refresh_state.entity_id', - 'final_entities.entity_id', - ) - .orderBy('refresh_state.entity_ref', 'asc'); // default sort + entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc'); // default sort } else { entitiesQuery.orderBy('final_entities.entity_id', 'asc'); // stable sort } @@ -326,16 +320,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const chunk of lodashChunk(request.entityRefs, 200)) { let query = this.database('final_entities') - .innerJoin( - 'refresh_state', - 'refresh_state.entity_id', - 'final_entities.entity_id', - ) .select({ - entityRef: 'refresh_state.entity_ref', + entityRef: 'final_entities.entity_ref', entity: 'final_entities.final_entity', }) - .whereIn('refresh_state.entity_ref', chunk); + .whereIn('final_entities.entity_ref', chunk); if (request?.filter) { query = parseFilter( @@ -343,7 +332,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query, this.database, false, - 'refresh_state.entity_id', + 'final_entities.entity_id', ); } @@ -669,11 +658,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async entityAncestry(rootRef: string): Promise { - const [rootRow] = await this.database('refresh_state') - .leftJoin('final_entities', { - 'refresh_state.entity_id': 'final_entities.entity_id', - }) - .where('refresh_state.entity_ref', '=', rootRef) + const [rootRow] = await this.database('final_entities') + .where('final_entities.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', }); @@ -698,16 +684,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const parentRows = await this.database( 'refresh_state_references', ) - .innerJoin('refresh_state', { - 'refresh_state_references.source_entity_ref': - 'refresh_state.entity_ref', - }) + .innerJoin('final_entities', { - 'refresh_state.entity_id': 'final_entities.entity_id', + 'refresh_state_references.source_entity_ref': + 'final_entities.entity_ref', }) .where('refresh_state_references.target_entity_ref', '=', currentRef) .select({ - parentEntityRef: 'refresh_state.entity_ref', + parentEntityRef: 'final_entities.entity_ref', parentEntityJson: 'final_entities.final_entity', }); diff --git a/plugins/catalog-backend/src/tests/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts index 873746ac2b..69e347bad0 100644 --- a/plugins/catalog-backend/src/tests/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -395,4 +395,79 @@ describe('migrations', () => { await knex.destroy(); }, ); + + it.each(databases.eachSupportedId())( + '20241024104700_add_entity_ref_to_final_entities.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20241024104700_add_entity_ref_to_final_entities.js', + ); + + await knex + .insert({ + entity_id: '8222246a-b572-49cf-a702-1a0fcfaae901', + entity_ref: 'k:ns/n', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }) + .into('refresh_state'); + + // Insert a simple URL before the migration + await knex + .insert({ + entity_id: '8222246a-b572-49cf-a702-1a0fcfaae901', + hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: '{}', + }) + .into('final_entities'); + + await migrateUpOnce(knex); + + // verify that the entity_ref column has been added + const columnInfo = await knex('final_entities').columnInfo(); + expect(columnInfo.entity_ref).not.toBeUndefined(); + + // verify that the contents of the entity_ref column are correct + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: '8222246a-b572-49cf-a702-1a0fcfaae901', + hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: '{}', + entity_ref: 'k:ns/n', + last_updated_at: null, + }, + ]), + ); + + await migrateDownOnce(knex); + + // verify that the entity_ref column has been removed + const revertedColumnInfo = await knex('final_entities').columnInfo(); + expect(revertedColumnInfo.entity_ref).toBeUndefined(); + + // verify that the contents are correct + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: '8222246a-b572-49cf-a702-1a0fcfaae901', + hash: '3f5a4d6ba8507be297bb7cd87c4b55b63e3f4c14', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: '{}', + last_updated_at: null, + }, + ]), + ); + + await knex.destroy(); + }, + ); });