catalog: relate the search table to final_entities, not refresh_state
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend': patch
|
||||
---
|
||||
|
||||
Changed the `search` table foreign key to point to `final_entities` instead of `refresh_state`
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright 2026 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* Changes the search table's foreign key from refresh_state(entity_id)
|
||||
* to final_entities(entity_id). This allows search entries to reference
|
||||
* final entities directly, with CASCADE delete when entities are removed.
|
||||
*
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.up = async function up(knex) {
|
||||
// Step 1: Drop the old foreign key constraint (to refresh_state)
|
||||
await knex.schema.alterTable('search', table => {
|
||||
table.dropForeign(['entity_id']);
|
||||
});
|
||||
|
||||
// Step 2: Delete orphaned rows where entity_id doesn't exist in final_entities
|
||||
await knex('search')
|
||||
.whereNotIn('entity_id', knex('final_entities').select('entity_id'))
|
||||
.delete();
|
||||
|
||||
// Step 3: Add new FK to final_entities(entity_id) with CASCADE
|
||||
await knex.schema.alterTable('search', table => {
|
||||
table
|
||||
.foreign('entity_id')
|
||||
.references('entity_id')
|
||||
.inTable('final_entities')
|
||||
.onDelete('CASCADE');
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.down = async function down(knex) {
|
||||
// Step 1: Drop FK to final_entities
|
||||
await knex.schema.alterTable('search', table => {
|
||||
table.dropForeign(['entity_id']);
|
||||
});
|
||||
|
||||
// Step 2: Delete orphaned rows where entity_id doesn't exist in refresh_state
|
||||
await knex('search')
|
||||
.whereNotIn('entity_id', knex('refresh_state').select('entity_id'))
|
||||
.delete();
|
||||
|
||||
// Step 3: Add back FK to refresh_state(entity_id) with CASCADE
|
||||
await knex.schema.alterTable('search', table => {
|
||||
table
|
||||
.foreign('entity_id')
|
||||
.references('entity_id')
|
||||
.inTable('refresh_state')
|
||||
.onDelete('CASCADE');
|
||||
});
|
||||
};
|
||||
@@ -23,12 +23,15 @@ import { CatalogScmEventsServiceSubscriber } from '@backstage/plugin-catalog-nod
|
||||
import { Knex } from 'knex';
|
||||
import { applyDatabaseMigrations } from '../database/migrations';
|
||||
import {
|
||||
DbFinalEntitiesRow,
|
||||
DbRefreshKeysRow,
|
||||
DbRefreshStateRow,
|
||||
DbSearchRow,
|
||||
} from '../database/tables';
|
||||
import { GenericScmEventRefreshProvider } from './GenericScmEventRefreshProvider';
|
||||
|
||||
jest.setTimeout(60_000);
|
||||
|
||||
describe('GenericScmEventRefreshProvider', () => {
|
||||
const databases = TestDatabases.create();
|
||||
|
||||
@@ -77,6 +80,16 @@ describe('GenericScmEventRefreshProvider', () => {
|
||||
});
|
||||
}
|
||||
|
||||
async function insertFinalEntity(knex: Knex, id: string) {
|
||||
await knex<DbFinalEntitiesRow>('final_entities').insert({
|
||||
entity_id: id,
|
||||
entity_ref: `k:ns/${id}`,
|
||||
hash: 'h',
|
||||
stitch_ticket: '',
|
||||
final_entity: '{}',
|
||||
});
|
||||
}
|
||||
|
||||
describe.each(databases.eachSupportedId())('%p', databaseId => {
|
||||
it('handles location.updated', async () => {
|
||||
const { knex, subscriber } = await initialize(databaseId);
|
||||
@@ -107,6 +120,11 @@ describe('GenericScmEventRefreshProvider', () => {
|
||||
},
|
||||
]);
|
||||
|
||||
// Insert final_entities for entities that will have search rows
|
||||
await insertFinalEntity(knex, '4');
|
||||
await insertFinalEntity(knex, '5');
|
||||
await insertFinalEntity(knex, '6');
|
||||
|
||||
await knex<DbSearchRow>('search').insert([
|
||||
// match exact blob in location
|
||||
{
|
||||
@@ -186,6 +204,13 @@ describe('GenericScmEventRefreshProvider', () => {
|
||||
},
|
||||
]);
|
||||
|
||||
// Insert final_entities for entities that will have search rows
|
||||
await insertFinalEntity(knex, '4');
|
||||
await insertFinalEntity(knex, '5');
|
||||
await insertFinalEntity(knex, '6');
|
||||
await insertFinalEntity(knex, '7');
|
||||
await insertFinalEntity(knex, '8');
|
||||
|
||||
await knex<DbSearchRow>('search').insert([
|
||||
// match blob in location
|
||||
{
|
||||
|
||||
@@ -70,6 +70,15 @@ describe('migrations', () => {
|
||||
last_discovery_at: new Date(),
|
||||
})
|
||||
.into('refresh_state');
|
||||
await knex
|
||||
.insert({
|
||||
entity_id: 'i1',
|
||||
hash: 'h',
|
||||
stitch_ticket: '',
|
||||
final_entity: '{}',
|
||||
entity_ref: 'k:ns/n1',
|
||||
})
|
||||
.into('final_entities');
|
||||
await knex
|
||||
.insert({ entity_id: 'i1', key: 'k1', value: 'v1' })
|
||||
.into('search');
|
||||
@@ -87,15 +96,6 @@ describe('migrations', () => {
|
||||
type: 't',
|
||||
})
|
||||
.into('relations');
|
||||
await knex
|
||||
.insert({
|
||||
entity_id: 'i1',
|
||||
hash: 'h',
|
||||
stitch_ticket: '',
|
||||
final_entity: '{}',
|
||||
entity_ref: 'k:ns/n1',
|
||||
})
|
||||
.into('final_entities');
|
||||
|
||||
await knex.delete().from('refresh_state').where({ entity_id: 'i1' });
|
||||
|
||||
@@ -690,4 +690,231 @@ describe('migrations', () => {
|
||||
await knex.destroy();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'20260214000000_search_fk_final_entities.js, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
|
||||
// Run migrations up to just before the target migration
|
||||
await migrateUntilBefore(
|
||||
knex,
|
||||
'20260214000000_search_fk_final_entities.js',
|
||||
);
|
||||
|
||||
// Insert rows into refresh_state
|
||||
await knex('refresh_state').insert([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
entity_ref: 'component:default/service-a',
|
||||
unprocessed_entity: '{}',
|
||||
processed_entity: '{}',
|
||||
errors: '[]',
|
||||
next_update_at: knex.fn.now(),
|
||||
last_discovery_at: knex.fn.now(),
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
entity_ref: 'component:default/service-b',
|
||||
unprocessed_entity: '{}',
|
||||
processed_entity: '{}',
|
||||
errors: '[]',
|
||||
next_update_at: knex.fn.now(),
|
||||
last_discovery_at: knex.fn.now(),
|
||||
},
|
||||
{
|
||||
entity_id: 'id3',
|
||||
entity_ref: 'api:default/my-api',
|
||||
unprocessed_entity: '{}',
|
||||
processed_entity: '{}',
|
||||
errors: '[]',
|
||||
next_update_at: knex.fn.now(),
|
||||
last_discovery_at: knex.fn.now(),
|
||||
},
|
||||
]);
|
||||
|
||||
// Insert rows into final_entities (only id1 and id2, not id3 - to test orphan deletion)
|
||||
await knex('final_entities').insert([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
entity_ref: 'component:default/service-a',
|
||||
hash: 'hash1',
|
||||
stitch_ticket: 'ticket1',
|
||||
final_entity: '{}',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
entity_ref: 'component:default/service-b',
|
||||
hash: 'hash2',
|
||||
stitch_ticket: 'ticket2',
|
||||
final_entity: '{}',
|
||||
},
|
||||
]);
|
||||
|
||||
// Insert rows into search table (with entity_id FK to refresh_state before migration)
|
||||
await knex('search').insert([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'metadata.name',
|
||||
value: 'service-a',
|
||||
original_value: 'service-a',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'metadata.name',
|
||||
value: 'service-b',
|
||||
original_value: 'service-b',
|
||||
},
|
||||
// id3 has no final_entities row, so this should be deleted during migration
|
||||
{ entity_id: 'id3', key: 'kind', value: 'api', original_value: 'API' },
|
||||
{
|
||||
entity_id: 'id3',
|
||||
key: 'metadata.name',
|
||||
value: 'my-api',
|
||||
original_value: 'my-api',
|
||||
},
|
||||
]);
|
||||
|
||||
// Verify initial state
|
||||
const preMigrationCount = await knex('search').count('* as count');
|
||||
expect(Number(preMigrationCount[0].count)).toBe(6);
|
||||
|
||||
// Run the migration
|
||||
await migrateUpOnce(knex);
|
||||
|
||||
// Verify orphaned rows (id3) were deleted since id3 is not in final_entities
|
||||
const postMigrationRows = await knex('search').orderBy([
|
||||
'entity_id',
|
||||
'key',
|
||||
]);
|
||||
expect(postMigrationRows).toEqual([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'metadata.name',
|
||||
value: 'service-a',
|
||||
original_value: 'service-a',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'metadata.name',
|
||||
value: 'service-b',
|
||||
original_value: 'service-b',
|
||||
},
|
||||
]);
|
||||
|
||||
// Verify FK cascade works: deleting from final_entities should cascade to search
|
||||
await knex('final_entities').where({ entity_id: 'id1' }).delete();
|
||||
const searchAfterDelete = await knex('search').orderBy([
|
||||
'entity_id',
|
||||
'key',
|
||||
]);
|
||||
expect(searchAfterDelete).toEqual([
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'metadata.name',
|
||||
value: 'service-b',
|
||||
original_value: 'service-b',
|
||||
},
|
||||
]);
|
||||
|
||||
// Restore id1 for down migration test
|
||||
await knex('final_entities').insert({
|
||||
entity_id: 'id1',
|
||||
entity_ref: 'component:default/service-a',
|
||||
hash: 'hash1',
|
||||
stitch_ticket: 'ticket1',
|
||||
final_entity: '{}',
|
||||
});
|
||||
await knex('search').insert([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
]);
|
||||
|
||||
// Run the down migration
|
||||
await migrateDownOnce(knex);
|
||||
|
||||
// Verify data is still intact after down migration
|
||||
const revertedSearchRows = await knex('search').orderBy([
|
||||
'entity_id',
|
||||
'key',
|
||||
]);
|
||||
expect(revertedSearchRows).toEqual([
|
||||
{
|
||||
entity_id: 'id1',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'metadata.name',
|
||||
value: 'service-b',
|
||||
original_value: 'service-b',
|
||||
},
|
||||
]);
|
||||
|
||||
// Verify FK is back to refresh_state: deleting from refresh_state should cascade
|
||||
await knex('refresh_state').where({ entity_id: 'id1' }).delete();
|
||||
const afterRefreshStateDelete = await knex('search').orderBy([
|
||||
'entity_id',
|
||||
'key',
|
||||
]);
|
||||
expect(afterRefreshStateDelete).toEqual([
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'kind',
|
||||
value: 'component',
|
||||
original_value: 'Component',
|
||||
},
|
||||
{
|
||||
entity_id: 'id2',
|
||||
key: 'metadata.name',
|
||||
value: 'service-b',
|
||||
original_value: 'service-b',
|
||||
},
|
||||
]);
|
||||
|
||||
await knex.destroy();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user