diff --git a/.changeset/swift-flowers-grin.md b/.changeset/swift-flowers-grin.md new file mode 100644 index 0000000000..33466615cd --- /dev/null +++ b/.changeset/swift-flowers-grin.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Changed the `search` table foreign key to point to `final_entities` instead of `refresh_state` diff --git a/plugins/catalog-backend/migrations/20260214000000_search_fk_final_entities.js b/plugins/catalog-backend/migrations/20260214000000_search_fk_final_entities.js new file mode 100644 index 0000000000..ed26716ca9 --- /dev/null +++ b/plugins/catalog-backend/migrations/20260214000000_search_fk_final_entities.js @@ -0,0 +1,69 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * Changes the search table's foreign key from refresh_state(entity_id) + * to final_entities(entity_id). This allows search entries to reference + * final entities directly, with CASCADE delete when entities are removed. + * + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Step 1: Drop the old foreign key constraint (to refresh_state) + await knex.schema.alterTable('search', table => { + table.dropForeign(['entity_id']); + }); + + // Step 2: Delete orphaned rows where entity_id doesn't exist in final_entities + await knex('search') + .whereNotIn('entity_id', knex('final_entities').select('entity_id')) + .delete(); + + // Step 3: Add new FK to final_entities(entity_id) with CASCADE + await knex.schema.alterTable('search', table => { + table + .foreign('entity_id') + .references('entity_id') + .inTable('final_entities') + .onDelete('CASCADE'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // Step 1: Drop FK to final_entities + await knex.schema.alterTable('search', table => { + table.dropForeign(['entity_id']); + }); + + // Step 2: Delete orphaned rows where entity_id doesn't exist in refresh_state + await knex('search') + .whereNotIn('entity_id', knex('refresh_state').select('entity_id')) + .delete(); + + // Step 3: Add back FK to refresh_state(entity_id) with CASCADE + await knex.schema.alterTable('search', table => { + table + .foreign('entity_id') + .references('entity_id') + .inTable('refresh_state') + .onDelete('CASCADE'); + }); +}; diff --git a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts index 8c74f350bb..fb54caab1b 100644 --- a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts +++ b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts @@ -23,12 +23,15 @@ import { CatalogScmEventsServiceSubscriber } from '@backstage/plugin-catalog-nod import { Knex } from 'knex'; import { applyDatabaseMigrations } from '../database/migrations'; import { + DbFinalEntitiesRow, DbRefreshKeysRow, DbRefreshStateRow, DbSearchRow, } from '../database/tables'; import { GenericScmEventRefreshProvider } from './GenericScmEventRefreshProvider'; +jest.setTimeout(60_000); + describe('GenericScmEventRefreshProvider', () => { const databases = TestDatabases.create(); @@ -77,6 +80,16 @@ describe('GenericScmEventRefreshProvider', () => { }); } + async function insertFinalEntity(knex: Knex, id: string) { + await knex('final_entities').insert({ + entity_id: id, + entity_ref: `k:ns/${id}`, + hash: 'h', + stitch_ticket: '', + final_entity: '{}', + }); + } + describe.each(databases.eachSupportedId())('%p', databaseId => { it('handles location.updated', async () => { const { knex, subscriber } = await initialize(databaseId); @@ -107,6 +120,11 @@ describe('GenericScmEventRefreshProvider', () => { }, ]); + // Insert final_entities for entities that will have search rows + await insertFinalEntity(knex, '4'); + await insertFinalEntity(knex, '5'); + await insertFinalEntity(knex, '6'); + await knex('search').insert([ // match exact blob in location { @@ -186,6 +204,13 @@ describe('GenericScmEventRefreshProvider', () => { }, ]); + // Insert final_entities for entities that will have search rows + await insertFinalEntity(knex, '4'); + await insertFinalEntity(knex, '5'); + await insertFinalEntity(knex, '6'); + await insertFinalEntity(knex, '7'); + await insertFinalEntity(knex, '8'); + await knex('search').insert([ // match blob in location { diff --git a/plugins/catalog-backend/src/tests/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts index 3807943b86..faf7351135 100644 --- a/plugins/catalog-backend/src/tests/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -70,6 +70,15 @@ describe('migrations', () => { last_discovery_at: new Date(), }) .into('refresh_state'); + await knex + .insert({ + entity_id: 'i1', + hash: 'h', + stitch_ticket: '', + final_entity: '{}', + entity_ref: 'k:ns/n1', + }) + .into('final_entities'); await knex .insert({ entity_id: 'i1', key: 'k1', value: 'v1' }) .into('search'); @@ -87,15 +96,6 @@ describe('migrations', () => { type: 't', }) .into('relations'); - await knex - .insert({ - entity_id: 'i1', - hash: 'h', - stitch_ticket: '', - final_entity: '{}', - entity_ref: 'k:ns/n1', - }) - .into('final_entities'); await knex.delete().from('refresh_state').where({ entity_id: 'i1' }); @@ -690,4 +690,231 @@ describe('migrations', () => { await knex.destroy(); }, ); + + it.each(databases.eachSupportedId())( + '20260214000000_search_fk_final_entities.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20260214000000_search_fk_final_entities.js', + ); + + // Insert rows into refresh_state + await knex('refresh_state').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/service-a', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + { + entity_id: 'id2', + entity_ref: 'component:default/service-b', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + { + entity_id: 'id3', + entity_ref: 'api:default/my-api', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + ]); + + // Insert rows into final_entities (only id1 and id2, not id3 - to test orphan deletion) + await knex('final_entities').insert([ + { + entity_id: 'id1', + entity_ref: 'component:default/service-a', + hash: 'hash1', + stitch_ticket: 'ticket1', + final_entity: '{}', + }, + { + entity_id: 'id2', + entity_ref: 'component:default/service-b', + hash: 'hash2', + stitch_ticket: 'ticket2', + final_entity: '{}', + }, + ]); + + // Insert rows into search table (with entity_id FK to refresh_state before migration) + await knex('search').insert([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id1', + key: 'metadata.name', + value: 'service-a', + original_value: 'service-a', + }, + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + // id3 has no final_entities row, so this should be deleted during migration + { entity_id: 'id3', key: 'kind', value: 'api', original_value: 'API' }, + { + entity_id: 'id3', + key: 'metadata.name', + value: 'my-api', + original_value: 'my-api', + }, + ]); + + // Verify initial state + const preMigrationCount = await knex('search').count('* as count'); + expect(Number(preMigrationCount[0].count)).toBe(6); + + // Run the migration + await migrateUpOnce(knex); + + // Verify orphaned rows (id3) were deleted since id3 is not in final_entities + const postMigrationRows = await knex('search').orderBy([ + 'entity_id', + 'key', + ]); + expect(postMigrationRows).toEqual([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id1', + key: 'metadata.name', + value: 'service-a', + original_value: 'service-a', + }, + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + ]); + + // Verify FK cascade works: deleting from final_entities should cascade to search + await knex('final_entities').where({ entity_id: 'id1' }).delete(); + const searchAfterDelete = await knex('search').orderBy([ + 'entity_id', + 'key', + ]); + expect(searchAfterDelete).toEqual([ + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + ]); + + // Restore id1 for down migration test + await knex('final_entities').insert({ + entity_id: 'id1', + entity_ref: 'component:default/service-a', + hash: 'hash1', + stitch_ticket: 'ticket1', + final_entity: '{}', + }); + await knex('search').insert([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + ]); + + // Run the down migration + await migrateDownOnce(knex); + + // Verify data is still intact after down migration + const revertedSearchRows = await knex('search').orderBy([ + 'entity_id', + 'key', + ]); + expect(revertedSearchRows).toEqual([ + { + entity_id: 'id1', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + ]); + + // Verify FK is back to refresh_state: deleting from refresh_state should cascade + await knex('refresh_state').where({ entity_id: 'id1' }).delete(); + const afterRefreshStateDelete = await knex('search').orderBy([ + 'entity_id', + 'key', + ]); + expect(afterRefreshStateDelete).toEqual([ + { + entity_id: 'id2', + key: 'kind', + value: 'component', + original_value: 'Component', + }, + { + entity_id: 'id2', + key: 'metadata.name', + value: 'service-b', + original_value: 'service-b', + }, + ]); + + await knex.destroy(); + }, + ); });