From 0e710fc837e6314620fe6327ecf68c4a82c6c039 Mon Sep 17 00:00:00 2001 From: kapaje Date: Tue, 1 Apr 2025 22:51:04 +0200 Subject: [PATCH 1/2] Update refresh_state table column for mysql databases Signed-off-by: kapaje --- .changeset/chubby-cougars-run.md | 6 +++ ...0401200503_update_refresh_state_columns.js | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 .changeset/chubby-cougars-run.md create mode 100644 plugins/catalog-backend/migrations/20250401200503_update_refresh_state_columns.js diff --git a/.changeset/chubby-cougars-run.md b/.changeset/chubby-cougars-run.md new file mode 100644 index 0000000000..deb954ef6b --- /dev/null +++ b/.changeset/chubby-cougars-run.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +This patch addresses an issue identified in Backstage when configured with a MySQL database. If an entity of type location +(e..all.yaml) has more than 70 referenced entities, clicking "Refresh" does not update the referenced entities as expected. This occurs because the TEXT type in MySQL has a limit of 65,535 bytes, which is insufficient to store all the referenced entities, causing the refresh operation to fail. diff --git a/plugins/catalog-backend/migrations/20250401200503_update_refresh_state_columns.js b/plugins/catalog-backend/migrations/20250401200503_update_refresh_state_columns.js new file mode 100644 index 0000000000..091dc896af --- /dev/null +++ b/plugins/catalog-backend/migrations/20250401200503_update_refresh_state_columns.js @@ -0,0 +1,43 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @param {import('knex').Knex} knex + * @returns {Promise} + */ +exports.up = async function up(knex) { + const isMySQL = knex.client.config.client.includes('mysql'); + // update the columns to longtext for MySQL + if (isMySQL) { + await knex.schema.alterTable('refresh_state', table => { + table.text('unprocessed_entity', 'longtext').alter(); + table.text('cache', 'longtext').alter(); + }); + } +}; + +/** + * @param {import('knex').Knex} knex + * @returns {Promise} + */ +exports.down = async function down(knex) { + const isMySQL = knex.client.config.client.includes('mysql'); + if (isMySQL) { + await knex.schema.alterTable('refresh_state', table => { + table.text('unprocessed_entity').alter(); + table.text('cache').alter(); + }); + } +}; From ab7f257755a7cba13b059a3fb17714e9c3a036e8 Mon Sep 17 00:00:00 2001 From: kapaje Date: Tue, 22 Apr 2025 19:59:23 +0200 Subject: [PATCH 2/2] add unit test regarding the database column type updata Signed-off-by: kapaje --- .../src/tests/migrations.test.ts | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/plugins/catalog-backend/src/tests/migrations.test.ts b/plugins/catalog-backend/src/tests/migrations.test.ts index bb840f7a21..f58b174f93 100644 --- a/plugins/catalog-backend/src/tests/migrations.test.ts +++ b/plugins/catalog-backend/src/tests/migrations.test.ts @@ -521,4 +521,87 @@ describe('migrations', () => { await knex.destroy(); }, ); + it.each(databases.eachSupportedId())( + '20250401200503_update_refresh_state_columns.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + // Run migrations up to just before the target migration + await migrateUntilBefore( + knex, + '20250401200503_update_refresh_state_columns.js', + ); + + // Insert a row with data into the refresh_state table + await knex('refresh_state').insert({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }); + + // Verify the data before the migration + const preMigrationData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(preMigrationData).toEqual( + expect.objectContaining({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + }), + ); + + // Run the migration + await migrateUpOnce(knex); + + // Verify the schema after the migration + const columnInfo = await knex('refresh_state').columnInfo(); + const expectedType = knex.client.config.client.includes('mysql') + ? 'longtext' + : 'text'; + expect(columnInfo.unprocessed_entity.type).toBe(expectedType); + expect(columnInfo.cache.type).toBe(expectedType); + + // Verify the data after the migration + const postMigrationData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(postMigrationData).toEqual( + expect.objectContaining({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + }), + ); + + // Roll back the migration + await migrateDownOnce(knex); + + // Verify the schema after rolling back + const revertedColumnInfo = await knex('refresh_state').columnInfo(); + expect(revertedColumnInfo.unprocessed_entity.type).toBe('text'); + expect(revertedColumnInfo.cache.type).toBe('text'); + + // Verify the data after rolling back + const postRollbackData = await knex('refresh_state') + .where({ entity_id: 'test-id' }) + .first(); + expect(postRollbackData).toEqual( + expect.objectContaining({ + entity_id: 'test-id', + entity_ref: 'k:ns/test', + unprocessed_entity: JSON.stringify({ key: 'value' }), + cache: JSON.stringify({ cacheKey: 'cacheValue' }), + }), + ); + + await knex.destroy(); + }, + ); });