From 4654a78dc1e9c415d5de25f3a9d31039c3da8c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 14 May 2025 15:44:02 +0200 Subject: [PATCH] update refresh_state_references.id to big int MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/tough-heads-knock.md | 5 ++ ...refresh_state_references_big_increments.js | 53 ++++++++++++ plugins/catalog-backend/report.sql.md | 14 +-- .../database/DefaultProviderDatabase.test.ts | 16 ++-- .../src/tests/migrations.test.ts | 86 +++++++++++++++++++ 5 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 .changeset/tough-heads-knock.md create mode 100644 plugins/catalog-backend/migrations/20250514000000_refresh_state_references_big_increments.js diff --git a/.changeset/tough-heads-knock.md b/.changeset/tough-heads-knock.md new file mode 100644 index 0000000000..da2e2c1163 --- /dev/null +++ b/.changeset/tough-heads-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Update `refresh_state_references.id` to be a big int diff --git a/plugins/catalog-backend/migrations/20250514000000_refresh_state_references_big_increments.js b/plugins/catalog-backend/migrations/20250514000000_refresh_state_references_big_increments.js new file mode 100644 index 0000000000..780f2f34a2 --- /dev/null +++ b/plugins/catalog-backend/migrations/20250514000000_refresh_state_references_big_increments.js @@ -0,0 +1,53 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param {import('knex').Knex} knex + * @returns {Promise} + */ +exports.up = async function up(knex) { + if (knex.client.config.client.includes('pg')) { + await knex.schema.raw( + `ALTER TABLE refresh_state_references ALTER COLUMN id TYPE bigint;`, + ); + await knex.schema.raw( + `ALTER SEQUENCE refresh_state_references_id_seq AS bigint MAXVALUE 9223372036854775807;`, + ); + } else if (knex.client.config.client.includes('mysql')) { + await knex.schema.raw( + `ALTER TABLE refresh_state_references MODIFY id bigint AUTO_INCREMENT;`, + ); + } +}; + +/** + * @param {import('knex').Knex} knex + * @returns {Promise} + */ +exports.down = async function down(knex) { + if (knex.client.config.client.includes('pg')) { + await knex.schema.raw( + `ALTER SEQUENCE refresh_state_references_id_seq AS integer MAXVALUE 2147483647;`, + ); + await knex.schema.raw( + `ALTER TABLE refresh_state_references ALTER COLUMN id TYPE integer;`, + ); + } else if (knex.client.config.client.includes('mysql')) { + await knex.schema.raw( + `ALTER TABLE refresh_state_references MODIFY id integer AUTO_INCREMENT;`, + ); + } +}; diff --git a/plugins/catalog-backend/report.sql.md b/plugins/catalog-backend/report.sql.md index 3402b05063..4f76d2a3f5 100644 --- a/plugins/catalog-backend/report.sql.md +++ b/plugins/catalog-backend/report.sql.md @@ -8,7 +8,7 @@ ## Sequences - `location_update_log_id_seq` (bigint) -- `refresh_state_references_id_seq` (integer) +- `refresh_state_references_id_seq` (bigint) ## Table `final_entities` @@ -93,12 +93,12 @@ ## Table `refresh_state_references` -| Column | Type | Nullable | Max Length | Default | -| ------------------- | --------- | -------- | ---------- | ------------------------------------------------------ | -| `id` | `integer` | false | - | `nextval('refresh_state_references_id_seq'::regclass)` | -| `source_entity_ref` | `text` | true | - | - | -| `source_key` | `text` | true | - | - | -| `target_entity_ref` | `text` | false | - | - | +| Column | Type | Nullable | Max Length | Default | +| ------------------- | -------- | -------- | ---------- | ------------------------------------------------------ | +| `id` | `bigint` | false | - | `nextval('refresh_state_references_id_seq'::regclass)` | +| `source_entity_ref` | `text` | true | - | - | +| `source_key` | `text` | true | - | - | +| `target_entity_ref` | `text` | false | - | - | ### Indices diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index 383e06100f..d9465b27a6 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -610,16 +610,18 @@ describe('DefaultProviderDatabase', () => { ); let references = await knex( 'refresh_state_references', - ).select(); + ) + .select() + .orderBy('id'); expect(references).toEqual([ { - id: 1, + id: expect.anything(), source_key: 'lols', source_entity_ref: null, target_entity_ref: 'component:default/a', }, { - id: 2, + id: expect.anything(), source_key: 'lols', source_entity_ref: null, target_entity_ref: 'component:default/b', @@ -670,16 +672,18 @@ describe('DefaultProviderDatabase', () => { ); references = await knex( 'refresh_state_references', - ).select(); + ) + .select() + .orderBy('id'); expect(references).toEqual([ { - id: 2, + id: expect.anything(), source_key: 'lols', source_entity_ref: null, target_entity_ref: 'component:default/b', }, { - id: 3, + id: expect.anything(), source_key: 'lols', source_entity_ref: null, target_entity_ref: 'component:default/a', diff --git a/plugins/catalog-backend/src/tests/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts index f58b174f93..e2a86c5c71 100644 --- a/plugins/catalog-backend/src/tests/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -521,6 +521,7 @@ describe('migrations', () => { await knex.destroy(); }, ); + it.each(databases.eachSupportedId())( '20250401200503_update_refresh_state_columns.js, %p', async databaseId => { @@ -604,4 +605,89 @@ describe('migrations', () => { await knex.destroy(); }, ); + + it.each(databases.eachSupportedId())( + '20250514000000_refresh_state_references_big_increments.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + const read = async () => { + return await knex('refresh_state_references') + .orderBy('id') + .then(rs => rs.map(r => ({ ...r, id: String(r.id) }))); + }; + + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20250514000000_refresh_state_references_big_increments.js', + ); + + await knex('refresh_state').insert({ + entity_id: 'a', + entity_ref: 'k:ns/a', + unprocessed_entity: '{}', + cache: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); + await knex('refresh_state_references').insert({ + source_key: 'before', + target_entity_ref: 'k:ns/a', + }); + + await migrateUpOnce(knex); + + // can still insert with auto generated id in sequence + await knex('refresh_state_references').insert({ + source_key: 'after1', + target_entity_ref: 'k:ns/a', + }); + await expect(read()).resolves.toEqual([ + { + id: '1', + source_key: 'before', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + { + id: '2', + source_key: 'after1', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + ]); + + await migrateDownOnce(knex); + + // can still insert with auto generated id in sequence + await knex('refresh_state_references').insert({ + source_key: 'after2', + target_entity_ref: 'k:ns/a', + }); + await expect(read()).resolves.toEqual([ + { + id: '1', + source_key: 'before', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + { + id: '2', + source_key: 'after1', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + { + id: '3', + source_key: 'after2', + source_entity_ref: null, + target_entity_ref: 'k:ns/a', + }, + ]); + + await knex.destroy(); + }, + ); });