diff --git a/.changeset/nice-pants-boil.md b/.changeset/nice-pants-boil.md new file mode 100644 index 0000000000..e8bdf0a00f --- /dev/null +++ b/.changeset/nice-pants-boil.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-catalog-backend': patch +--- + +Adds MySQL support for the catalog-backend diff --git a/packages/backend-common/src/database/util.ts b/packages/backend-common/src/database/util.ts index 0a7d3cd7f7..f5abcd7fba 100644 --- a/packages/backend-common/src/database/util.ts +++ b/packages/backend-common/src/database/util.ts @@ -29,6 +29,7 @@ export function isDatabaseConflictError(e: unknown) { typeof message === 'string' && (/SQLITE_CONSTRAINT(?:_UNIQUE)?: UNIQUE/.test(message) || /UNIQUE constraint failed:/.test(message) || - /unique constraint/.test(message)) + /unique constraint/.test(message) || + /Duplicate entry/.test(message)) // MySQL uniqueness error msg ); } diff --git a/plugins/catalog-backend/migrations/20200511113813_init.js b/plugins/catalog-backend/migrations/20200511113813_init.js index aa2ff6affa..a34912135e 100644 --- a/plugins/catalog-backend/migrations/20200511113813_init.js +++ b/plugins/catalog-backend/migrations/20200511113813_init.js @@ -59,7 +59,7 @@ exports.up = async function up(knex) { 'An opaque string that changes for each update operation to any part of the entity, including metadata.', ); table - .string('generation') + .integer('generation') .notNullable() .unsigned() .comment( diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js index 86e0e48e40..37aee4d985 100644 --- a/plugins/catalog-backend/migrations/20200702153613_entities.js +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -31,6 +31,7 @@ exports.up = async function up(knex) { } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); + table.dropForeign(['location_id']); }); // Setup temporary tables await knex.schema.renameTable('entities_search', 'tmp_entities_search'); @@ -56,7 +57,7 @@ exports.up = async function up(knex) { 'An opaque string that changes for each update operation to any part of the entity, including metadata.', ); table - .string('generation') + .integer('generation') .notNullable() .unsigned() .comment( diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js index 9c7d966087..5acfbeb9ba 100644 --- a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -23,7 +23,7 @@ exports.up = async function up(knex) { // Sqlite does not support alter column. if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { - table.text('value').nullable().alter({ alterType: true }); + table.string('value').nullable().alter({ alterType: true }); }); } }; diff --git a/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js index 01be4789d0..9f0cabac41 100644 --- a/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js +++ b/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js @@ -24,8 +24,8 @@ exports.up = async function up(knex) { .where({ namespace: null }) .update({ namespace: 'default' }); await knex('entities_search').update({ - key: knex.raw('LOWER(key)'), - value: knex.raw('LOWER(value)'), + key: knex.raw('LOWER(??)', ['key']), + value: knex.raw('LOWER(??)', ['value']), }); }; diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 941b8a0a3f..412791d8d7 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -21,19 +21,20 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities', table => { - table.text('full_name').nullable(); + table.string('full_name').nullable(); }); await knex('entities').update({ full_name: knex.raw( - "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", + "LOWER(??) || ':' || LOWER(COALESCE(??, 'default')) || '/' || LOWER(??)", + ['kind', 'namespace', 'name'], ), }); // SQLite does not support alter column if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { - table.text('full_name').notNullable().alter({ alterNullable: true }); + table.string('full_name').notNullable().alter({ alterNullable: true }); }); } diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js index 47097fc788..6e7ae9b8fe 100644 --- a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -31,7 +31,8 @@ exports.up = async function up(knex) { // apiVersion and kind should not contain any JSON unsafe chars, and both // metadata and spec are already valid serialized JSON data: knex.raw( - `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, + `'{"apiVersion":"' || ?? || '","kind":"' || ?? || '","metadata":' || ?? || COALESCE(',"spec":' || ??, '') || '}'`, + ['api_version', 'kind', 'metadata', 'spec'], ), }); diff --git a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js index 996c69de8c..313eb88600 100644 --- a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js +++ b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js @@ -24,7 +24,7 @@ exports.up = async function up(knex) { // sqlite doesn't support dropPrimary so we recreate it properly instead await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table .uuid('originating_entity_id') .references('id') @@ -61,7 +61,7 @@ exports.down = async function down(knex) { if (knex.client.config.client.includes('sqlite3')) { await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table .uuid('originating_entity_id') .references('id') diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index da276940fa..1c8081c0ef 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -20,39 +20,30 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { + const isMySQL = knex.client.config.client.includes('mysql'); await knex.schema.createTable('refresh_state', table => { - table.comment( - 'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.', - ); + table.comment('Location refresh states'); table - .text('entity_id') + .string('entity_id') .primary() .notNullable() - .comment( - 'Primary ID, which will also be used as the uid of the resulting entity', - ); + .comment('Primary ID, also used as the uid of the entity'); table - .text('entity_ref') + .string('entity_ref') .notNullable() - .comment('A reference to the entity that the refresh state is tied to'); + .comment('A reference to the entity for this refresh state'); table .text('unprocessed_entity') .notNullable() - .comment( - 'The unprocessed entity (in its source form, before being run through all of the processors) as JSON', - ); + .comment('The unprocessed entity (in original form) as JSON'); table .text('processed_entity') .nullable() - .comment( - 'The processed entity (after running through all processors, but before being stitched together with state and relations) as JSON', - ); + .comment('The processed entity (not yet stitched) as JSON'); table .text('cache') .nullable() - .comment( - 'Cache information tied to the refreshing of this entity, such as etag information or actual response caching', - ); + .comment('Cache information tied to refreshes of this entity'); table .text('errors') .notNullable() @@ -64,7 +55,7 @@ exports.up = async function up(knex) { table .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() - .comment('The last timestamp of which this entity was discovered'); + .comment('The last timestamp that this entity was discovered'); table.unique(['entity_ref'], { indexName: 'refresh_state_entity_ref_uniq', }); @@ -74,31 +65,23 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('final_entities', table => { - table.comment( - 'This table contains the final entity result after processing and stitching', - ); + table.comment('Final entities after processing and stitching'); table - .text('entity_id') + .string('entity_id') .primary() .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') - .comment( - 'Entity ID which corresponds to the ID in the refresh_state table', - ); + .comment('Entity ID -> refresh_state table'); table - .text('hash') + .string('hash') .notNullable() - .comment( - 'Stable hash of the entity data, to be used for caching and avoiding redundant work', - ); + .comment('Stable hash of the entity data'); table .text('stitch_ticket') .notNullable() - .comment( - 'A random value representing a unique stitch attempt ticket, that gets updated each time that a stitching attempt is made on the entity', - ); + .comment('Random value representing a unique stitch attempt ticket'); table .text('final_entity') .nullable() @@ -107,29 +90,24 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('refresh_state_references', table => { - table.comment( - 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', - ); + const textColumn = isMySQL + ? table.string.bind(table) + : table.text.bind(table); + + table.comment('Edges between refresh state rows'); table .increments('id') .comment('Primary key to distinguish unique lines from each other'); - table - .text('source_key') + textColumn('source_key') .nullable() - .comment( - 'When the reference source is not an entity, this is an opaque identifier for that source.', - ); - table - .text('source_entity_ref') + .comment('Opaque identifier for non-entity sources'); + textColumn('source_entity_ref') .nullable() .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') - .comment( - 'When the reference source is an entity, this is the EntityRef of the source entity.', - ); - table - .text('target_entity_ref') + .comment('EntityRef of entity sources'); + textColumn('target_entity_ref') .notNullable() .references('entity_ref') .inTable('refresh_state') @@ -147,36 +125,34 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table - .text('originating_entity_id') + .string('originating_entity_id') .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .notNullable() .comment('The entity that provided the relation'); table - .text('source_entity_ref') + .string('source_entity_ref') .notNullable() - .comment('The entity reference of the source entity of the relation'); + .comment('Entity reference of the source entity of the relation'); table - .text('type') + .string('type') .notNullable() .comment('The type of the relation between the entities'); table - .text('target_entity_ref') + .string('target_entity_ref') .notNullable() - .comment('The entity reference of the target entity of the relation'); + .comment('Entity reference of the target entity of the relation'); table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); await knex.schema.createTable('search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); + table.comment('Flattened key-values from the entities, for filtering'); table - .text('entity_id') + .string('entity_id') .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') diff --git a/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js b/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js index 37ab5e12cb..f15e2c208c 100644 --- a/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js +++ b/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js @@ -24,9 +24,7 @@ exports.up = async function up(knex) { table .text('location_key') .nullable() - .comment( - 'An opaque key that uniquely identifies the location of an entity in order to support conflict resolution', - ); + .comment('Opaque conflict resolution key'); }); }; diff --git a/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js b/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js index 7d7a4ac578..136be9899e 100644 --- a/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js +++ b/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js @@ -24,7 +24,7 @@ exports.up = async function up(knex) { table .text('unprocessed_hash') .nullable() - .comment('A hash of the unprocessed contents, used to detect changes'); + .comment('A hash of the unprocessed contents'); }); }; diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index b1b67f68f2..ddc39d4005 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -23,14 +23,14 @@ exports.up = async function up(knex) { 'This table contains relations between entities and keys to trigger refreshes with', ); table - .text('entity_id') + .string('entity_id') .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); table - .text('key') + .string('key') .notNullable() .comment( 'A reference to a key which should be used to trigger a refresh on this entity', diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index f766589fe7..1056557706 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -36,7 +36,7 @@ import { generateStableHash } from './util'; describe('Default Processing Database', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createDatabase( diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 4a30ecaec6..97259bce29 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -15,7 +15,7 @@ */ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; -import { ConflictError, isError, NotFoundError } from '@backstage/errors'; +import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; @@ -82,6 +82,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { refreshKeys, locationKey, } = options; + const configClient = tx.client.config.client; const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), @@ -114,10 +115,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Delete old relations // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries. let previousRelationRows: DbRelationsRow[]; - if ( - tx.client.config.client.includes('sqlite3') || - tx.client.config.client.includes('mysql') - ) { + if (configClient.includes('sqlite3') || configClient.includes('mysql')) { previousRelationRows = await tx('relations') .select('*') .where({ originating_entity_id: id }); @@ -663,11 +661,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { last_discovery_at: tx.fn.now(), }); - // TODO(Rugvip): only tested towards Postgres and SQLite + // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. // We have to do this because the only way to detect if there was a conflict with // SQLite is to catch the error, while Postgres needs to ignore the conflict to not // break the ongoing transaction. - if (!tx.client.config.client.includes('sqlite3')) { + if (tx.client.config.client.includes('pg')) { query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime } @@ -675,14 +673,15 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const result: { rowCount?: number; length?: number } = await query; return result.rowCount === 1 || result.length === 1; } catch (error) { - // SQLite reached this rather than the rowCount check above - if ( - isError(error) && - error.message.includes('UNIQUE constraint failed') - ) { + // SQLite, or MySQL reached this rather than the rowCount check above + if (!isDatabaseConflictError(error)) { + throw error; + } else { + this.options.logger.debug( + `Unable to insert a new refresh state row, ${error}`, + ); return false; } - throw error; } } diff --git a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index 2a9b1f8d65..9a957f8838 100644 --- a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -20,7 +20,7 @@ import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createLocationStore(databaseId: TestDatabaseId) { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 2c75785dad..b84596c78c 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -31,7 +31,7 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; describe('DefaultEntitiesCatalog', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); const stitch = jest.fn(); const stitcher: Stitcher = { stitch } as any; diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index c82a17cb6f..14f22f862b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -277,6 +277,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async removeEntityByUid(uid: string): Promise { + const dbConfig = this.database.client.config; + // Clear the hashed state of the immediate parents of the deleted entity. // This makes sure that when they get reprocessed, their output is written // down again. The reason for wanting to do this, is that if the user @@ -285,21 +287,53 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { // means it'll never try to write down the children again (it assumes that // they already exist). This means that without the code below, the database // never "heals" from accidental deletes. - await this.database('refresh_state') - .update({ - result_hash: 'child-was-deleted', - next_update_at: this.database.fn.now(), - }) - .whereIn('entity_ref', function parents(builder) { - return builder - .from('refresh_state') - .innerJoin('refresh_state_references', { - 'refresh_state_references.target_entity_ref': - 'refresh_state.entity_ref', - }) - .where('refresh_state.entity_id', '=', uid) - .select('refresh_state_references.source_entity_ref'); - }); + if (dbConfig.client.includes('mysql')) { + // MySQL doesn't support the syntax we need to do this in a single query, + // http://dev.mysql.com/doc/refman/5.6/en/update.html + const results = await this.database('refresh_state') + .select('entity_id') + .whereIn('entity_ref', function parents(builder) { + return builder + .from('refresh_state') + .innerJoin( + 'refresh_state_references', + { + 'refresh_state_references.target_entity_ref': + 'refresh_state.entity_ref', + }, + ) + .where('refresh_state.entity_id', '=', uid) + .select('refresh_state_references.source_entity_ref'); + }); + await this.database('refresh_state') + .update({ + result_hash: 'child-was-deleted', + next_update_at: this.database.fn.now(), + }) + .whereIn( + 'entity_id', + results.map(key => key.entity_id), + ); + } else { + await this.database('refresh_state') + .update({ + result_hash: 'child-was-deleted', + next_update_at: this.database.fn.now(), + }) + .whereIn('entity_ref', function parents(builder) { + return builder + .from('refresh_state') + .innerJoin( + 'refresh_state_references', + { + 'refresh_state_references.target_entity_ref': + 'refresh_state.entity_ref', + }, + ) + .where('refresh_state.entity_id', '=', uid) + .select('refresh_state_references.source_entity_ref'); + }); + } // Stitch the entities that the deleted one had relations to. If we do not // do this, the entities in the other end of the relations will still look @@ -324,7 +358,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .select({ ref: 'relations.source_entity_ref' }), ); - // Perform the actual deletion await this.database('refresh_state') .where('entity_id', uid) .delete(); diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 419aedc15d..3ed4f3fe36 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -36,7 +36,7 @@ import { DefaultRefreshService } from './DefaultRefreshService'; describe('Refresh integration', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createDatabase( diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index b42e68d105..a3657e45fd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -29,7 +29,7 @@ import { Stitcher } from './Stitcher'; describe('Stitcher', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); const logger = getVoidLogger();