From 20a5161f045922c3f87a8a26453e295a58da74b0 Mon Sep 17 00:00:00 2001 From: tonedef Date: Tue, 30 Aug 2022 15:11:59 -0700 Subject: [PATCH 01/82] Adding MySQL support to the catalog-backend plugin Signed-off-by: tonedef --- .changeset/nice-pants-boil.md | 6 ++ packages/backend-common/src/database/util.ts | 4 +- .../migrations/20200511113813_init.js | 2 +- .../migrations/20200702153613_entities.js | 3 +- .../20200923104503_case_insensitivity.js | 4 +- .../20201005122705_add_entity_full_name.js | 7 +- .../20201006130744_entity_data_column.js | 3 +- .../20201007201501_index_entity_search.js | 13 ++- .../20201123205611_relations_table_uniq.js | 4 +- .../20210302150147_refresh_state.js | 93 +++++++------------ ...210622104022_refresh_state_location_key.js | 4 +- ...0925102509_add_refresh_state_input_hash.js | 2 +- .../migrations/20220616202842_refresh_keys.js | 4 +- .../DefaultProcessingDatabase.test.ts | 12 ++- .../src/database/DefaultProcessingDatabase.ts | 15 ++- .../modules/core/DefaultLocationStore.test.ts | 2 +- .../service/DefaultEntitiesCatalog.test.ts | 7 +- .../src/service/DefaultEntitiesCatalog.ts | 61 ++++++++---- .../src/service/DefaultRefreshService.test.ts | 2 +- .../src/stitching/Stitcher.test.ts | 2 +- 20 files changed, 135 insertions(+), 115 deletions(-) create mode 100644 .changeset/nice-pants-boil.md diff --git a/.changeset/nice-pants-boil.md b/.changeset/nice-pants-boil.md new file mode 100644 index 0000000000..e8bdf0a00f --- /dev/null +++ b/.changeset/nice-pants-boil.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-catalog-backend': patch +--- + +Adds MySQL support for the catalog-backend diff --git a/packages/backend-common/src/database/util.ts b/packages/backend-common/src/database/util.ts index 0a7d3cd7f7..a4905d54b0 100644 --- a/packages/backend-common/src/database/util.ts +++ b/packages/backend-common/src/database/util.ts @@ -29,6 +29,8 @@ export function isDatabaseConflictError(e: unknown) { typeof message === 'string' && (/SQLITE_CONSTRAINT(?:_UNIQUE)?: UNIQUE/.test(message) || /UNIQUE constraint failed:/.test(message) || - /unique constraint/.test(message)) + /unique constraint/.test(message) || + /Duplicate entry/.test(message) // MySQL uniqueness error msg + ) ); } diff --git a/plugins/catalog-backend/migrations/20200511113813_init.js b/plugins/catalog-backend/migrations/20200511113813_init.js index aa2ff6affa..a34912135e 100644 --- a/plugins/catalog-backend/migrations/20200511113813_init.js +++ b/plugins/catalog-backend/migrations/20200511113813_init.js @@ -59,7 +59,7 @@ exports.up = async function up(knex) { 'An opaque string that changes for each update operation to any part of the entity, including metadata.', ); table - .string('generation') + .integer('generation') .notNullable() .unsigned() .comment( diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js index 86e0e48e40..37aee4d985 100644 --- a/plugins/catalog-backend/migrations/20200702153613_entities.js +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -31,6 +31,7 @@ exports.up = async function up(knex) { } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); + table.dropForeign(['location_id']); }); // Setup temporary tables await knex.schema.renameTable('entities_search', 'tmp_entities_search'); @@ -56,7 +57,7 @@ exports.up = async function up(knex) { 'An opaque string that changes for each update operation to any part of the entity, including metadata.', ); table - .string('generation') + .integer('generation') .notNullable() .unsigned() .comment( diff --git a/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js b/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js index 01be4789d0..9f0cabac41 100644 --- a/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js +++ b/plugins/catalog-backend/migrations/20200923104503_case_insensitivity.js @@ -24,8 +24,8 @@ exports.up = async function up(knex) { .where({ namespace: null }) .update({ namespace: 'default' }); await knex('entities_search').update({ - key: knex.raw('LOWER(key)'), - value: knex.raw('LOWER(value)'), + key: knex.raw('LOWER(??)', ['key']), + value: knex.raw('LOWER(??)', ['value']), }); }; diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 941b8a0a3f..8705ae26a6 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -21,19 +21,20 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities', table => { - table.text('full_name').nullable(); + table.string('full_name').nullable(); }); await knex('entities').update({ full_name: knex.raw( - "LOWER(kind) || ':' || LOWER(COALESCE(namespace, 'default')) || '/' || LOWER(name)", + "LOWER(??) || ':' || LOWER(COALESCE(??, 'default')) || '/' || LOWER(??)", + ['kind', 'namespace', 'name'], ), }); // SQLite does not support alter column if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { - table.text('full_name').notNullable().alter({ alterNullable: true }); + table.string('full_name').notNullable().alter(); }); } diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js index 47097fc788..6e7ae9b8fe 100644 --- a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -31,7 +31,8 @@ exports.up = async function up(knex) { // apiVersion and kind should not contain any JSON unsafe chars, and both // metadata and spec are already valid serialized JSON data: knex.raw( - `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`, + `'{"apiVersion":"' || ?? || '","kind":"' || ?? || '","metadata":' || ?? || COALESCE(',"spec":' || ??, '') || '}'`, + ['api_version', 'kind', 'metadata', 'spec'], ), }); diff --git a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js index 59fd34b700..3611c91b3d 100644 --- a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js +++ b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js @@ -21,8 +21,17 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities_search', table => { - table.index(['key'], 'entities_search_key'); - table.index(['value'], 'entities_search_value'); + if (knex.client.config.client.includes('mysql')) { + table.index(['key'], 'entities_search_key', { + indexType: 'FULLTEXT', + }); + table.index(['value'], 'entities_search_value', { + indexType: 'FULLTEXT', + }); + } else { + table.index(['key'], 'entities_search_key'); + table.index(['value'], 'entities_search_value'); + } }); }; diff --git a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js index 996c69de8c..313eb88600 100644 --- a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js +++ b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js @@ -24,7 +24,7 @@ exports.up = async function up(knex) { // sqlite doesn't support dropPrimary so we recreate it properly instead await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table .uuid('originating_entity_id') .references('id') @@ -61,7 +61,7 @@ exports.down = async function down(knex) { if (knex.client.config.client.includes('sqlite3')) { await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table .uuid('originating_entity_id') .references('id') diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index da276940fa..3a58196127 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -21,38 +21,28 @@ */ exports.up = async function up(knex) { await knex.schema.createTable('refresh_state', table => { - table.comment( - 'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.', - ); + table.comment('Location refresh states'); table - .text('entity_id') + .string('entity_id') .primary() .notNullable() - .comment( - 'Primary ID, which will also be used as the uid of the resulting entity', - ); + .comment('Primary ID, also used as the uid of the entity'); table - .text('entity_ref') + .string('entity_ref') .notNullable() - .comment('A reference to the entity that the refresh state is tied to'); + .comment('A reference to the entity for this refresh state'); table .text('unprocessed_entity') .notNullable() - .comment( - 'The unprocessed entity (in its source form, before being run through all of the processors) as JSON', - ); + .comment('The unprocessed entity (in original form) as JSON'); table .text('processed_entity') .nullable() - .comment( - 'The processed entity (after running through all processors, but before being stitched together with state and relations) as JSON', - ); + .comment('The processed entity (not yet stitched) as JSON'); table .text('cache') .nullable() - .comment( - 'Cache information tied to the refreshing of this entity, such as etag information or actual response caching', - ); + .comment('Cache information tied to refreshes of this entity'); table .text('errors') .notNullable() @@ -64,41 +54,28 @@ exports.up = async function up(knex) { table .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() - .comment('The last timestamp of which this entity was discovered'); - table.unique(['entity_ref'], { - indexName: 'refresh_state_entity_ref_uniq', - }); + .comment('The last timestamp that this entity was discovered'); + table.unique(['entity_ref'], 'refresh_state_entity_ref_uniq'); table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); }); await knex.schema.createTable('final_entities', table => { - table.comment( - 'This table contains the final entity result after processing and stitching', - ); + table.comment('Final entities after processing and stitching'); table - .text('entity_id') + .string('entity_id') .primary() .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') - .comment( - 'Entity ID which corresponds to the ID in the refresh_state table', - ); + .comment('Entity ID -> refresh_state table'); + table.text('hash').notNullable().comment('Stable hash of the entity data'); table - .text('hash') + .string('stitch_ticket') .notNullable() - .comment( - 'Stable hash of the entity data, to be used for caching and avoiding redundant work', - ); - table - .text('stitch_ticket') - .notNullable() - .comment( - 'A random value representing a unique stitch attempt ticket, that gets updated each time that a stitching attempt is made on the entity', - ); + .comment('Random value representing a unique stitch attempt ticket'); table .text('final_entity') .nullable() @@ -107,29 +84,23 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('refresh_state_references', table => { - table.comment( - 'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.', - ); + table.comment('Edges between refresh state rows'); table .increments('id') .comment('Primary key to distinguish unique lines from each other'); table - .text('source_key') + .string('source_key') .nullable() - .comment( - 'When the reference source is not an entity, this is an opaque identifier for that source.', - ); + .comment('Opaque identifier for non-entity sources'); table - .text('source_entity_ref') + .string('source_entity_ref') .nullable() .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') - .comment( - 'When the reference source is an entity, this is the EntityRef of the source entity.', - ); + .comment('EntityRef of entity sources'); table - .text('target_entity_ref') + .string('target_entity_ref') .notNullable() .references('entity_ref') .inTable('refresh_state') @@ -147,36 +118,34 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('relations', table => { - table.comment('All relations between entities in the catalog'); + table.comment('All relations between entities'); table - .text('originating_entity_id') + .string('originating_entity_id') .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .notNullable() .comment('The entity that provided the relation'); table - .text('source_entity_ref') + .string('source_entity_ref') .notNullable() - .comment('The entity reference of the source entity of the relation'); + .comment('Entity reference of the source entity of the relation'); table - .text('type') + .string('type') .notNullable() .comment('The type of the relation between the entities'); table - .text('target_entity_ref') + .string('target_entity_ref') .notNullable() - .comment('The entity reference of the target entity of the relation'); + .comment('Entity reference of the target entity of the relation'); table.index('source_entity_ref', 'relations_source_entity_ref_idx'); table.index('originating_entity_id', 'relations_source_entity_id_idx'); }); await knex.schema.createTable('search', table => { - table.comment( - 'Flattened key-values from the entities, used for quick filtering', - ); + table.comment('Flattened key-values from the entities, for filtering'); table - .text('entity_id') + .string('entity_id') .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') diff --git a/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js b/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js index 37ab5e12cb..f15e2c208c 100644 --- a/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js +++ b/plugins/catalog-backend/migrations/20210622104022_refresh_state_location_key.js @@ -24,9 +24,7 @@ exports.up = async function up(knex) { table .text('location_key') .nullable() - .comment( - 'An opaque key that uniquely identifies the location of an entity in order to support conflict resolution', - ); + .comment('Opaque conflict resolution key'); }); }; diff --git a/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js b/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js index 7d7a4ac578..136be9899e 100644 --- a/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js +++ b/plugins/catalog-backend/migrations/20210925102509_add_refresh_state_input_hash.js @@ -24,7 +24,7 @@ exports.up = async function up(knex) { table .text('unprocessed_hash') .nullable() - .comment('A hash of the unprocessed contents, used to detect changes'); + .comment('A hash of the unprocessed contents'); }); }; diff --git a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js index b1b67f68f2..ddc39d4005 100644 --- a/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js +++ b/plugins/catalog-backend/migrations/20220616202842_refresh_keys.js @@ -23,14 +23,14 @@ exports.up = async function up(knex) { 'This table contains relations between entities and keys to trigger refreshes with', ); table - .text('entity_id') + .string('entity_id') .notNullable() .references('entity_id') .inTable('refresh_state') .onDelete('CASCADE') .comment('A reference to the entity that the refresh key is tied to'); table - .text('key') + .string('key') .notNullable() .comment( 'A reference to a key which should be used to trigger a refresh on this entity', diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 46b327733d..e8d538fb0b 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -36,7 +36,7 @@ import { generateStableHash } from './util'; describe('Default Processing Database', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createDatabase( @@ -1447,10 +1447,12 @@ describe('Default Processing Database', () => { const result1 = await db.transaction(async tx => db.listParents(tx, { entityRef: 'component:default/foobar' }), ); - expect(result1.entityRefs).toEqual([ - 'location:default/root-1', - 'location:default/root-2', - ]); + expect(result1.entityRefs).toEqual( + expect.arrayContaining([ + 'location:default/root-1', + 'location:default/root-2', + ]), + ); const result2 = await db.transaction(async tx => db.listParents(tx, { entityRef: 'location:default/root-1' }), diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 3dc5325178..2ffe69725f 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -82,6 +82,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { refreshKeys, locationKey, } = options; + const configClient = tx.client.config.client; const refreshResult = await tx('refresh_state') .update({ processed_entity: JSON.stringify(processedEntity), @@ -114,10 +115,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Delete old relations // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries. let previousRelationRows: DbRelationsRow[]; - if ( - tx.client.config.client.includes('sqlite3') || - tx.client.config.client.includes('mysql') - ) { + if (configClient.includes('sqlite3') || configClient.includes('mysql')) { previousRelationRows = await tx('relations') .select('*') .where({ originating_entity_id: id }); @@ -663,11 +661,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { last_discovery_at: tx.fn.now(), }); - // TODO(Rugvip): only tested towards Postgres and SQLite + // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. // We have to do this because the only way to detect if there was a conflict with // SQLite is to catch the error, while Postgres needs to ignore the conflict to not // break the ongoing transaction. - if (!tx.client.config.client.includes('sqlite3')) { + if (tx.client.config.client.includes('pg')) { query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime } @@ -675,10 +673,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const result: { rowCount?: number; length?: number } = await query; return result.rowCount === 1 || result.length === 1; } catch (error) { - // SQLite reached this rather than the rowCount check above + // SQLite, or MySQL reached this rather than the rowCount check above if ( isError(error) && - error.message.includes('UNIQUE constraint failed') + (error.message.includes('UNIQUE constraint failed') || + error.message.includes('Duplicate entry')) // MySQL failure ) { return false; } diff --git a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index 2a9b1f8d65..9a957f8838 100644 --- a/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -20,7 +20,7 @@ import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createLocationStore(databaseId: TestDatabaseId) { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 49d4fe9e19..306c141a62 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -30,7 +30,7 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; describe('DefaultEntitiesCatalog', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); const stitch = jest.fn(); const stitcher: Stitcher = { stitch } as any; @@ -239,7 +239,10 @@ describe('DefaultEntitiesCatalog', () => { expect.arrayContaining([ { entity: expect.objectContaining({ metadata: { name: 'root' } }), - parentEntityRefs: ['k:default/parent1', 'k:default/parent2'], + parentEntityRefs: expect.arrayContaining([ + 'k:default/parent1', + 'k:default/parent2', + ]), }, { entity: expect.objectContaining({ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index dc84b12484..90e40377f2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -238,6 +238,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async removeEntityByUid(uid: string): Promise { + const dbConfig = this.database.client.config; + // Clear the hashed state of the immediate parents of the deleted entity. // This makes sure that when they get reprocessed, their output is written // down again. The reason for wanting to do this, is that if the user @@ -246,21 +248,49 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { // means it'll never try to write down the children again (it assumes that // they already exist). This means that without the code below, the database // never "heals" from accidental deletes. - await this.database('refresh_state') - .update({ - result_hash: 'child-was-deleted', - next_update_at: this.database.fn.now(), - }) - .whereIn('entity_ref', function parents(builder) { - return builder - .from('refresh_state') - .innerJoin('refresh_state_references', { - 'refresh_state_references.target_entity_ref': - 'refresh_state.entity_ref', - }) - .where('refresh_state.entity_id', '=', uid) - .select('refresh_state_references.source_entity_ref'); - }); + if (dbConfig.client.includes('mysql')) { + // MySQL doesn't support the syntax we need to do this in a single query, + // http://dev.mysql.com/doc/refman/5.6/en/update.html + const results = await this.database('refresh_state') + .select('entity_id') + .whereIn('entity_ref', function parents(builder) { + return builder + .from('refresh_state') + .innerJoin( + 'refresh_state_references', + { + 'refresh_state_references.target_entity_ref': + 'refresh_state.entity_ref', + }, + ) + .where('refresh_state.entity_id', '=', uid) + .select('refresh_state_references.source_entity_ref'); + }); + await this.database('refresh_state') + .update({ result_hash: 'child-was-deleted' }) + .whereIn( + 'entity_id', + results.map(key => key.entity_id), + ); + } else { + await this.database('refresh_state') + .update({ + result_hash: 'child-was-deleted', + }) + .whereIn('entity_ref', function parents(builder) { + return builder + .from('refresh_state') + .innerJoin( + 'refresh_state_references', + { + 'refresh_state_references.target_entity_ref': + 'refresh_state.entity_ref', + }, + ) + .where('refresh_state.entity_id', '=', uid) + .select('refresh_state_references.source_entity_ref'); + }); + } // Stitch the entities that the deleted one had relations to. If we do not // do this, the entities in the other end of the relations will still look @@ -285,7 +315,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .select({ ref: 'relations.source_entity_ref' }), ); - // Perform the actual deletion await this.database('refresh_state') .where('entity_id', uid) .delete(); diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 419aedc15d..3ed4f3fe36 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -36,7 +36,7 @@ import { DefaultRefreshService } from './DefaultRefreshService'; describe('Refresh integration', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); async function createDatabase( diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index 9accebf1fc..c6827892ee 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -29,7 +29,7 @@ import { Stitcher } from './Stitcher'; describe('Stitcher', () => { const databases = TestDatabases.create({ - ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); const logger = getVoidLogger(); From 1e62ef8675904d7148d5e675141880e22b41faff Mon Sep 17 00:00:00 2001 From: tonedef Date: Tue, 30 Aug 2022 15:51:52 -0700 Subject: [PATCH 02/82] fix parentheses Signed-off-by: tonedef --- packages/backend-common/src/database/util.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/backend-common/src/database/util.ts b/packages/backend-common/src/database/util.ts index a4905d54b0..f5abcd7fba 100644 --- a/packages/backend-common/src/database/util.ts +++ b/packages/backend-common/src/database/util.ts @@ -30,7 +30,6 @@ export function isDatabaseConflictError(e: unknown) { (/SQLITE_CONSTRAINT(?:_UNIQUE)?: UNIQUE/.test(message) || /UNIQUE constraint failed:/.test(message) || /unique constraint/.test(message) || - /Duplicate entry/.test(message) // MySQL uniqueness error msg - ) + /Duplicate entry/.test(message)) // MySQL uniqueness error msg ); } From 7a854747522c55dcdacc4bc61107e222211c40aa Mon Sep 17 00:00:00 2001 From: tonedef Date: Thu, 6 Oct 2022 15:14:57 -0700 Subject: [PATCH 03/82] address feedback Signed-off-by: tonedef --- .../20210302150147_refresh_state.js | 28 +++++++++++++------ .../DefaultProcessingDatabase.test.ts | 10 +++---- .../src/database/DefaultProcessingDatabase.ts | 12 ++++---- .../service/DefaultEntitiesCatalog.test.ts | 5 +--- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index 3a58196127..bf1611a099 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -20,6 +20,11 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { + let STRING_TEXT = 'text'; + if (knex.client.config.client.includes('mysql')) { + STRING_TEXT = 'string'; + } + await knex.schema.createTable('refresh_state', table => { table.comment('Location refresh states'); table @@ -55,7 +60,9 @@ exports.up = async function up(knex) { .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp that this entity was discovered'); - table.unique(['entity_ref'], 'refresh_state_entity_ref_uniq'); + table.unique(['entity_ref'], { + indexName: 'refresh_state_entity_ref_uniq', + }); table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); @@ -71,9 +78,12 @@ exports.up = async function up(knex) { .inTable('refresh_state') .onDelete('CASCADE') .comment('Entity ID -> refresh_state table'); - table.text('hash').notNullable().comment('Stable hash of the entity data'); table - .string('stitch_ticket') + .string('hash') + .notNullable() + .comment('Stable hash of the entity data'); + table + .text('stitch_ticket') .notNullable() .comment('Random value representing a unique stitch attempt ticket'); table @@ -88,19 +98,19 @@ exports.up = async function up(knex) { table .increments('id') .comment('Primary key to distinguish unique lines from each other'); - table - .string('source_key') + // @ts-ignore + table[STRING_TEXT]('source_key') .nullable() .comment('Opaque identifier for non-entity sources'); - table - .string('source_entity_ref') + // @ts-ignore + table[STRING_TEXT]('source_entity_ref') .nullable() .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment('EntityRef of entity sources'); - table - .string('target_entity_ref') + // @ts-ignore + table[STRING_TEXT]('target_entity_ref') .notNullable() .references('entity_ref') .inTable('refresh_state') diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index e8d538fb0b..3375629d0b 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -1447,12 +1447,10 @@ describe('Default Processing Database', () => { const result1 = await db.transaction(async tx => db.listParents(tx, { entityRef: 'component:default/foobar' }), ); - expect(result1.entityRefs).toEqual( - expect.arrayContaining([ - 'location:default/root-1', - 'location:default/root-2', - ]), - ); + expect(result1.entityRefs).toEqual([ + 'location:default/root-1', + 'location:default/root-2', + ]); const result2 = await db.transaction(async tx => db.listParents(tx, { entityRef: 'location:default/root-1' }), diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 2ffe69725f..319ccae7f8 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -674,14 +674,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { return result.rowCount === 1 || result.length === 1; } catch (error) { // SQLite, or MySQL reached this rather than the rowCount check above - if ( - isError(error) && - (error.message.includes('UNIQUE constraint failed') || - error.message.includes('Duplicate entry')) // MySQL failure - ) { + if (!isDatabaseConflictError(error)) { + throw error; + } else { + this.options.logger.debug( + `Unable to insert a new refresh state row, ${error}`, + ); return false; } - throw error; } } diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 306c141a62..049619c831 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -239,10 +239,7 @@ describe('DefaultEntitiesCatalog', () => { expect.arrayContaining([ { entity: expect.objectContaining({ metadata: { name: 'root' } }), - parentEntityRefs: expect.arrayContaining([ - 'k:default/parent1', - 'k:default/parent2', - ]), + parentEntityRefs: ['k:default/parent1', 'k:default/parent2'], }, { entity: expect.objectContaining({ From 7e87a56a0f3309074ed05a2356af0b7e7b52ff51 Mon Sep 17 00:00:00 2001 From: tonedef Date: Thu, 6 Oct 2022 15:23:32 -0700 Subject: [PATCH 04/82] fix ts error Signed-off-by: tonedef --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 319ccae7f8..fda65e35a1 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -15,7 +15,7 @@ */ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; -import { ConflictError, isError, NotFoundError } from '@backstage/errors'; +import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; From 1fdd91514a45b16d782418c3775cd007771e57fd Mon Sep 17 00:00:00 2001 From: tonedef Date: Thu, 6 Oct 2022 16:26:32 -0700 Subject: [PATCH 05/82] repush indexName change Signed-off-by: tonedef --- .../migrations/20210302150147_refresh_state.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index bf1611a099..1dd82749ed 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -60,9 +60,7 @@ exports.up = async function up(knex) { .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp that this entity was discovered'); - table.unique(['entity_ref'], { - indexName: 'refresh_state_entity_ref_uniq', - }); + table.unique(['entity_ref'], { indexName: 'refresh_state_entity_ref_uniq' }); table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); From 1882a59951bbdf0d4d53ecea1bf359a56f72bf24 Mon Sep 17 00:00:00 2001 From: tonedef Date: Thu, 6 Oct 2022 16:38:18 -0700 Subject: [PATCH 06/82] repush changes Signed-off-by: tonedef --- .../migrations/20210302150147_refresh_state.js | 6 ++++-- .../src/database/DefaultProcessingDatabase.test.ts | 2 +- .../src/database/DefaultProcessingDatabase.ts | 2 +- .../src/service/DefaultEntitiesCatalog.test.ts | 2 +- .../catalog-backend/src/service/DefaultEntitiesCatalog.ts | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index 1dd82749ed..dadc656e86 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -60,7 +60,9 @@ exports.up = async function up(knex) { .dateTime('last_discovery_at') // TODO: timezone or change to epoch-millis or similar .notNullable() .comment('The last timestamp that this entity was discovered'); - table.unique(['entity_ref'], { indexName: 'refresh_state_entity_ref_uniq' }); + table.unique(['entity_ref'], { + indexName: 'refresh_state_entity_ref_uniq', + }); table.index('entity_id', 'refresh_state_entity_id_idx'); table.index('entity_ref', 'refresh_state_entity_ref_idx'); table.index('next_update_at', 'refresh_state_next_update_at_idx'); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 3375629d0b..f10ac3df6e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index fda65e35a1..d739c9c8ee 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 049619c831..591c69bcc7 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 90e40377f2..2b9a3f3310 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 50ae38e537fc2075f5dbc06c2eec1bf7d8c311f2 Mon Sep 17 00:00:00 2001 From: tonedef Date: Fri, 7 Oct 2022 09:24:11 -0700 Subject: [PATCH 07/82] fix notice issue Signed-off-by: tonedef --- .../catalog-backend/migrations/20210302150147_refresh_state.js | 2 +- .../src/database/DefaultProcessingDatabase.test.ts | 2 +- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- .../catalog-backend/src/service/DefaultEntitiesCatalog.test.ts | 2 +- plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index dadc656e86..bf1611a099 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index f10ac3df6e..3375629d0b 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index d739c9c8ee..fda65e35a1 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 591c69bcc7..049619c831 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 2b9a3f3310..90e40377f2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 112c6d5605813936f044ef2ffa0995b2e49576f6 Mon Sep 17 00:00:00 2001 From: tonedef Date: Fri, 28 Oct 2022 13:54:42 -0700 Subject: [PATCH 08/82] address feedback Signed-off-by: tonedef --- .../20201005122705_add_entity_full_name.js | 2 +- .../20201007201501_index_entity_search.js | 22 +++++++++---------- .../20210302150147_refresh_state.js | 19 +++++++--------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 8705ae26a6..412791d8d7 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -34,7 +34,7 @@ exports.up = async function up(knex) { // SQLite does not support alter column if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { - table.string('full_name').notNullable().alter(); + table.string('full_name').notNullable().alter({ alterNullable: true }); }); } diff --git a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js index 3611c91b3d..be700a324a 100644 --- a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js +++ b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js @@ -21,17 +21,17 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities_search', table => { - if (knex.client.config.client.includes('mysql')) { - table.index(['key'], 'entities_search_key', { - indexType: 'FULLTEXT', - }); - table.index(['value'], 'entities_search_value', { - indexType: 'FULLTEXT', - }); - } else { - table.index(['key'], 'entities_search_key'); - table.index(['value'], 'entities_search_value'); - } + const options = knex.client.config.client.includes('mysql') ? { indexType: 'FULLTEXT', } : {} + table.index( + ['key'], + 'entities_search_key', + options + ); + table.index( + ['value'], + 'entities_search_value', + options + ); }); }; diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index bf1611a099..c3c3dc3f12 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -20,11 +20,7 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - let STRING_TEXT = 'text'; - if (knex.client.config.client.includes('mysql')) { - STRING_TEXT = 'string'; - } - + const isMySQL = knex.client.config.client.includes('mysql'); await knex.schema.createTable('refresh_state', table => { table.comment('Location refresh states'); table @@ -94,23 +90,24 @@ exports.up = async function up(knex) { }); await knex.schema.createTable('refresh_state_references', table => { + const textColumn = isMySQL + ? table.string.bind(table) + : table.text.bind(table); + table.comment('Edges between refresh state rows'); table .increments('id') .comment('Primary key to distinguish unique lines from each other'); - // @ts-ignore - table[STRING_TEXT]('source_key') + textColumn('source_key') .nullable() .comment('Opaque identifier for non-entity sources'); - // @ts-ignore - table[STRING_TEXT]('source_entity_ref') + textColumn('source_entity_ref') .nullable() .references('entity_ref') .inTable('refresh_state') .onDelete('CASCADE') .comment('EntityRef of entity sources'); - // @ts-ignore - table[STRING_TEXT]('target_entity_ref') + textColumn('target_entity_ref') .notNullable() .references('entity_ref') .inTable('refresh_state') From 1f41007d85c02db247178cb9f8dedeccd51047a1 Mon Sep 17 00:00:00 2001 From: tonedef Date: Sat, 29 Oct 2022 17:07:18 -0700 Subject: [PATCH 09/82] prettier changes Signed-off-by: tonedef --- .../20201007201501_index_entity_search.js | 16 +++++----------- .../migrations/20210302150147_refresh_state.js | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js index be700a324a..ea26067a06 100644 --- a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js +++ b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js @@ -21,17 +21,11 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities_search', table => { - const options = knex.client.config.client.includes('mysql') ? { indexType: 'FULLTEXT', } : {} - table.index( - ['key'], - 'entities_search_key', - options - ); - table.index( - ['value'], - 'entities_search_value', - options - ); + const options = knex.client.config.client.includes('mysql') + ? { indexType: 'FULLTEXT' } + : {}; + table.index(['key'], 'entities_search_key', options); + table.index(['value'], 'entities_search_value', options); }); }; diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js index c3c3dc3f12..1c8081c0ef 100644 --- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js +++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js @@ -91,8 +91,8 @@ exports.up = async function up(knex) { await knex.schema.createTable('refresh_state_references', table => { const textColumn = isMySQL - ? table.string.bind(table) - : table.text.bind(table); + ? table.string.bind(table) + : table.text.bind(table); table.comment('Edges between refresh state rows'); table From ed29fad6dae02f914d6793929c8c64a3948b6f0c Mon Sep 17 00:00:00 2001 From: tonedef Date: Tue, 1 Nov 2022 10:27:33 -0700 Subject: [PATCH 10/82] Empty commit to trigger GHA Signed-off-by: tonedef From 9516b0c355c1bf745a4f62adbbca75a9b07eaf92 Mon Sep 17 00:00:00 2001 From: Nikita Karpukhin Date: Mon, 28 Nov 2022 16:32:51 +0100 Subject: [PATCH 11/82] Add support for sending virtual pageviews on search events Signed-off-by: Nikita Karpukhin --- .changeset/selfish-lizards-invent.md | 5 + plugins/analytics-module-ga/README.md | 25 +++++ plugins/analytics-module-ga/config.d.ts | 26 ++++++ .../AnalyticsApi/GoogleAnalytics.test.ts | 93 +++++++++++++++++++ .../AnalyticsApi/GoogleAnalytics.ts | 25 ++++- .../src/util/VirtualSearchPageView.ts | 43 +++++++++ 6 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 .changeset/selfish-lizards-invent.md create mode 100644 plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts diff --git a/.changeset/selfish-lizards-invent.md b/.changeset/selfish-lizards-invent.md new file mode 100644 index 0000000000..0eb9e110cf --- /dev/null +++ b/.changeset/selfish-lizards-invent.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-analytics-module-ga': minor +--- + +Added support for sending virtual pageviews on `search` events diff --git a/plugins/analytics-module-ga/README.md b/plugins/analytics-module-ga/README.md index 4b822ea3e3..605044f37c 100644 --- a/plugins/analytics-module-ga/README.md +++ b/plugins/analytics-module-ga/README.md @@ -169,6 +169,31 @@ export const apis: AnyApiFactory[] = [ ]; ``` +### Enabling Site Search + +If you wish to see all of the search events in the [Site Search](https://support.google.com/analytics/answer/1012264) +section of Google Analytics, you can enable sending virtual pageviews on every `search` event like so: + +```yaml +app: + analytics: + ga: + virtualSearchPageView: + mode: only # Defaults to 'disabled' + mountPath: /virtual-search # Defaults to '/search' + queryParam: term # Defaults to 'query' +``` + +Available `mode`s are: + +- `disabled` - no virtual pageviews are sent, default behavior +- `only` - sends virtual pageviews _instead_ of `search` events +- `both` - sends both virtual pageviews _and_ `search` events + +Virtual pageviews will be sent to the path specified in the `mountPath`, and the search term will be +set as the value for query parameter `queryParam`, e.g. the example config above will result in +virtual pageviews being sent to `/virtual-search?term=SearchTermHere`. + ### Debugging and Testing In pre-production environments, you may wish to set additional configurations diff --git a/plugins/analytics-module-ga/config.d.ts b/plugins/analytics-module-ga/config.d.ts index b91f5812c1..bf9c3b304b 100644 --- a/plugins/analytics-module-ga/config.d.ts +++ b/plugins/analytics-module-ga/config.d.ts @@ -53,6 +53,32 @@ export interface Config { */ identity?: 'disabled' | 'optional' | 'required'; + /** + * Controls whether to send virtual pageviews on `search` events. + * Can be used to enable Site Search in GA. + */ + virtualSearchPageView?: { + /** + * - `disabled`: (Default) no virtual pageviews are sent + * - `only`: Sends virtual pageview _instead_ of the `search` event + * - `both`: Sends both the `search` event _and_ the virtual pageview + * @visibility frontend + */ + mode?: 'disabled' | 'only' | 'both'; + /** + * Specifies on which path the main Search page is mounted. + * Defaults to `/search`. + * @visibility frontend + */ + mountPath?: string; + /** + * Specifies which query param is used in the virtual pageview URL. + * Defaults to `query`. + * @visibility frontend + */ + queryParam?: string; + }; + /** * Whether or not to log analytics debug statements to the console. * Defaults to false. diff --git a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts index c9c2a02c69..8dbf06ed0f 100644 --- a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts +++ b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts @@ -152,6 +152,99 @@ describe('GoogleAnalytics', () => { }); }); + it('captures virtual pageviews', () => { + const config = new ConfigReader({ + app: { + analytics: { + ga: { + trackingId, + testMode: true, + virtualSearchPageView: { mode: 'only' }, + }, + }, + }, + }); + const api = GoogleAnalytics.fromConfig(config); + api.captureEvent({ + action: 'search', + subject: 'test search', + context, + }); + + const [command, data] = ReactGA.testModeAPI.calls[1]; + expect(command).toBe('send'); + expect(data).toMatchObject({ + hitType: 'pageview', + page: '/search?query=test search', + }); + }); + + it('captures virtual pageviews alongside search events', () => { + const config = new ConfigReader({ + app: { + analytics: { + ga: { + trackingId, + testMode: true, + virtualSearchPageView: { mode: 'both' }, + }, + }, + }, + }); + const api = GoogleAnalytics.fromConfig(config); + api.captureEvent({ + action: 'search', + subject: 'test search', + context, + }); + + const [pageviewCommand, pageViewData] = ReactGA.testModeAPI.calls[1]; + expect(pageviewCommand).toBe('send'); + expect(pageViewData).toMatchObject({ + hitType: 'pageview', + page: '/search?query=test search', + }); + const [searchCommand, searchData] = ReactGA.testModeAPI.calls[2]; + expect(searchCommand).toBe('send'); + expect(searchData).toMatchObject({ + hitType: 'event', + eventCategory: context.extension, + eventAction: 'search', + eventLabel: 'test search', + }); + }); + + it('captures virtual pageviews on custom route with custom query param', () => { + const config = new ConfigReader({ + app: { + analytics: { + ga: { + trackingId, + testMode: true, + virtualSearchPageView: { + mode: 'only', + mountPath: '/custom', + queryParam: 'term', + }, + }, + }, + }, + }); + const api = GoogleAnalytics.fromConfig(config); + api.captureEvent({ + action: 'search', + subject: 'test search', + context, + }); + + const [command, data] = ReactGA.testModeAPI.calls[1]; + expect(command).toBe('send'); + expect(data).toMatchObject({ + hitType: 'pageview', + page: '/custom?term=test search', + }); + }); + it('captures configured custom dimensions/metrics on events', () => { const api = GoogleAnalytics.fromConfig(advancedConfig); diff --git a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts index 498b118d0d..16e4807008 100644 --- a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts +++ b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts @@ -18,12 +18,16 @@ import ReactGA from 'react-ga'; import { AnalyticsApi, AnalyticsContextValue, - AnalyticsEventAttributes, AnalyticsEvent, + AnalyticsEventAttributes, IdentityApi, } from '@backstage/core-plugin-api'; import { Config } from '@backstage/config'; import { DeferredCapture } from '../../../util'; +import { + parseVirtualSearchPageViewConfig, + VirtualSearchPageViewConfig, +} from '../../../util/VirtualSearchPageView'; type CustomDimensionOrMetricConfig = { type: 'dimension' | 'metric'; @@ -40,6 +44,7 @@ export class GoogleAnalytics implements AnalyticsApi { private readonly cdmConfig: CustomDimensionOrMetricConfig[]; private customUserIdTransform?: (userEntityRef: string) => Promise; private readonly capture: DeferredCapture; + private readonly virtualSearchPageView: VirtualSearchPageViewConfig; /** * Instantiate the implementation and initialize ReactGA. @@ -51,6 +56,7 @@ export class GoogleAnalytics implements AnalyticsApi { identity: string; trackingId: string; scriptSrc?: string; + virtualSearchPageView: VirtualSearchPageViewConfig; testMode: boolean; debug: boolean; }) { @@ -61,11 +67,13 @@ export class GoogleAnalytics implements AnalyticsApi { identityApi, userIdTransform = 'sha-256', scriptSrc, + virtualSearchPageView, testMode, debug, } = options; this.cdmConfig = cdmConfig; + this.virtualSearchPageView = virtualSearchPageView; // Initialize Google Analytics. ReactGA.initialize(trackingId, { @@ -105,6 +113,9 @@ export class GoogleAnalytics implements AnalyticsApi { const scriptSrc = config.getOptionalString('app.analytics.ga.scriptSrc'); const identity = config.getOptionalString('app.analytics.ga.identity') || 'disabled'; + const virtualSearchPageView = parseVirtualSearchPageViewConfig( + config.getOptionalConfig('app.analytics.ga.virtualSearchPageView'), + ); const debug = config.getOptionalBoolean('app.analytics.ga.debug') ?? false; const testMode = config.getOptionalBoolean('app.analytics.ga.testMode') ?? false; @@ -134,6 +145,7 @@ export class GoogleAnalytics implements AnalyticsApi { identity, trackingId, scriptSrc, + virtualSearchPageView, cdmConfig, testMode, debug, @@ -154,6 +166,17 @@ export class GoogleAnalytics implements AnalyticsApi { return; } + if (this.virtualSearchPageView.mode !== 'disabled' && action === 'search') { + const { mountPath, queryParam } = this.virtualSearchPageView; + this.capture.pageview( + `${mountPath}?${queryParam}=${subject}`, + customMetadata, + ); + if (this.virtualSearchPageView.mode === 'only') { + return; + } + } + this.capture.event({ category: context.extension || 'App', action, diff --git a/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts b/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts new file mode 100644 index 0000000000..8da2eebca3 --- /dev/null +++ b/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config } from '@backstage/config'; + +type VirtualSearchPageViewType = 'disabled' | 'only' | 'both'; + +export type VirtualSearchPageViewConfig = { + mode: VirtualSearchPageViewType; + mountPath: string; + queryParam: string; +}; + +function isVirtualSearchPageViewType( + value: string | undefined, +): value is VirtualSearchPageViewType { + return value === 'disabled' || value === 'only' || value === 'both'; +} + +export function parseVirtualSearchPageViewConfig( + config: Config | undefined, +): VirtualSearchPageViewConfig { + const vspvModeString = config?.getOptionalString('mode'); + return { + mode: isVirtualSearchPageViewType(vspvModeString) + ? vspvModeString + : 'disabled', + mountPath: config?.getOptionalString('mountPath') ?? '/search', + queryParam: config?.getOptionalString('queryParam') ?? 'query', + }; +} From b7e918f3b8c2c644dbdb8e1133afe5c53ee0f4bc Mon Sep 17 00:00:00 2001 From: Nikita Karpukhin Date: Tue, 29 Nov 2022 11:30:12 +0100 Subject: [PATCH 12/82] Change version bump from minor to patch, link to README.md Signed-off-by: Nikita Karpukhin --- .changeset/selfish-lizards-invent.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.changeset/selfish-lizards-invent.md b/.changeset/selfish-lizards-invent.md index 0eb9e110cf..fd486416d8 100644 --- a/.changeset/selfish-lizards-invent.md +++ b/.changeset/selfish-lizards-invent.md @@ -1,5 +1,6 @@ --- -'@backstage/plugin-analytics-module-ga': minor +'@backstage/plugin-analytics-module-ga': patch --- -Added support for sending virtual pageviews on `search` events +Added support for sending virtual pageviews on `search` events in order to enable +Site Search functionality in GA. For more information consult [README](/plugins/analytics-module-ga/README.md#enabling-site-search) From 73ab069f5e3e1020f658d5a5d0b9f1558f6b0100 Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Tue, 29 Nov 2022 13:47:04 -0500 Subject: [PATCH 13/82] update Stack Overflow plugin to support API Access Token Signed-off-by: Connor Younglund --- plugins/stack-overflow-backend/README.md | 1 + .../search/StackOverflowQuestionsCollatorFactory.ts | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/plugins/stack-overflow-backend/README.md b/plugins/stack-overflow-backend/README.md index a35a6678cb..65fa50e062 100644 --- a/plugins/stack-overflow-backend/README.md +++ b/plugins/stack-overflow-backend/README.md @@ -23,6 +23,7 @@ If you have a private stack overflow instance you will need to supply an API key stackoverflow: baseUrl: https://api.stackexchange.com/2.2 # alternative: your internal stack overflow instance apiKey: $STACK_OVERFLOW_API_KEY + apiAccessToken: $STACK_OVERFLOW_API_ACCESS_TOKEN ``` ## Areas of Responsibility diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 1aa0e5d300..68aa83cdeb 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -52,6 +52,7 @@ export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; maxPage?: number; apiKey?: string; + apiAccessToken?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; }; @@ -67,6 +68,7 @@ export class StackOverflowQuestionsCollatorFactory protected requestParams: StackOverflowQuestionsRequestParams; private readonly baseUrl: string | undefined; private readonly apiKey: string | undefined; + private readonly apiAccessToken: string | undefined; private readonly maxPage: number | undefined; private readonly logger: Logger; public readonly type: string = 'stack-overflow'; @@ -74,6 +76,7 @@ export class StackOverflowQuestionsCollatorFactory private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) { this.baseUrl = options.baseUrl; this.apiKey = options.apiKey; + this.apiAccessToken = options.apiAccessToken; this.maxPage = options.maxPage; this.requestParams = options.requestParams; this.logger = options.logger.child({ documentType: this.type }); @@ -84,6 +87,7 @@ export class StackOverflowQuestionsCollatorFactory options: StackOverflowQuestionsCollatorFactoryOptions, ) { const apiKey = config.getOptionalString('stackoverflow.apiKey'); + const apiAccessToken = config.getOptionalString('stackoverflow.apiAccessToken'); const baseUrl = config.getOptionalString('stackoverflow.baseUrl') || 'https://api.stackexchange.com/2.2'; @@ -93,6 +97,7 @@ export class StackOverflowQuestionsCollatorFactory baseUrl, maxPage, apiKey, + apiAccessToken, }); } @@ -138,6 +143,11 @@ export class StackOverflowQuestionsCollatorFactory } const res = await fetch( `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, + this.apiAccessToken ? { + headers: { + 'X-API-Access-Token': this.apiAccessToken, + }, + } : undefined, ); const data = await res.json(); From fd0ca6f447ecfc58855d90c7a90682568662d57f Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Tue, 29 Nov 2022 13:58:51 -0500 Subject: [PATCH 14/82] add changeset Signed-off-by: Connor Younglund --- .changeset/short-turtles-dream.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/short-turtles-dream.md diff --git a/.changeset/short-turtles-dream.md b/.changeset/short-turtles-dream.md new file mode 100644 index 0000000000..9192f9137c --- /dev/null +++ b/.changeset/short-turtles-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow-backend': minor +--- + +Added option to supply API Access Token From ea857549b53fb065ae5ab772fbde40e5f8b24d4a Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Tue, 29 Nov 2022 14:18:23 -0500 Subject: [PATCH 15/82] format with Prettier Signed-off-by: Connor Younglund --- .../StackOverflowQuestionsCollatorFactory.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 68aa83cdeb..ebbad99d0a 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -87,7 +87,9 @@ export class StackOverflowQuestionsCollatorFactory options: StackOverflowQuestionsCollatorFactoryOptions, ) { const apiKey = config.getOptionalString('stackoverflow.apiKey'); - const apiAccessToken = config.getOptionalString('stackoverflow.apiAccessToken'); + const apiAccessToken = config.getOptionalString( + 'stackoverflow.apiAccessToken', + ); const baseUrl = config.getOptionalString('stackoverflow.baseUrl') || 'https://api.stackexchange.com/2.2'; @@ -143,11 +145,13 @@ export class StackOverflowQuestionsCollatorFactory } const res = await fetch( `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, - this.apiAccessToken ? { - headers: { - 'X-API-Access-Token': this.apiAccessToken, - }, - } : undefined, + this.apiAccessToken + ? { + headers: { + 'X-API-Access-Token': this.apiAccessToken, + }, + } + : undefined, ); const data = await res.json(); From 6756292c2499ea6de1f96dffba442c056120bd92 Mon Sep 17 00:00:00 2001 From: Connor Younglund Date: Tue, 29 Nov 2022 14:52:42 -0500 Subject: [PATCH 16/82] generate updated api-report.md Signed-off-by: Connor Younglund --- plugins/stack-overflow-backend/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/stack-overflow-backend/api-report.md b/plugins/stack-overflow-backend/api-report.md index 11bfbaadc9..f6c7530de9 100644 --- a/plugins/stack-overflow-backend/api-report.md +++ b/plugins/stack-overflow-backend/api-report.md @@ -43,6 +43,7 @@ export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; maxPage?: number; apiKey?: string; + apiAccessToken?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; }; From da94f4f1a58cf6ab86dbb7d39c41aaa2d36cb9a0 Mon Sep 17 00:00:00 2001 From: tonedef Date: Tue, 29 Nov 2022 20:28:59 -0800 Subject: [PATCH 17/82] rm FULLTEXT, change value to string, add next_update_at Signed-off-by: tonedef --- .../migrations/20200807120600_entitySearch.js | 2 +- .../migrations/20201007201501_index_entity_search.js | 7 ++----- .../catalog-backend/src/service/DefaultEntitiesCatalog.ts | 6 +++++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js index 9c7d966087..5acfbeb9ba 100644 --- a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -23,7 +23,7 @@ exports.up = async function up(knex) { // Sqlite does not support alter column. if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { - table.text('value').nullable().alter({ alterType: true }); + table.string('value').nullable().alter({ alterType: true }); }); } }; diff --git a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js index ea26067a06..59fd34b700 100644 --- a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js +++ b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js @@ -21,11 +21,8 @@ */ exports.up = async function up(knex) { await knex.schema.alterTable('entities_search', table => { - const options = knex.client.config.client.includes('mysql') - ? { indexType: 'FULLTEXT' } - : {}; - table.index(['key'], 'entities_search_key', options); - table.index(['value'], 'entities_search_value', options); + table.index(['key'], 'entities_search_key'); + table.index(['value'], 'entities_search_value'); }); }; diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 90e40377f2..4c31f10d1f 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -267,7 +267,10 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .select('refresh_state_references.source_entity_ref'); }); await this.database('refresh_state') - .update({ result_hash: 'child-was-deleted' }) + .update({ + result_hash: 'child-was-deleted', + next_update_at: this.database.fn.now(), + }) .whereIn( 'entity_id', results.map(key => key.entity_id), @@ -276,6 +279,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { await this.database('refresh_state') .update({ result_hash: 'child-was-deleted', + next_update_at: this.database.fn.now(), }) .whereIn('entity_ref', function parents(builder) { return builder From 623fd9aff9f1713fd5ebf09069cbfd5cfc84b1bf Mon Sep 17 00:00:00 2001 From: Nikita Karpukhin Date: Wed, 30 Nov 2022 09:18:49 +0100 Subject: [PATCH 18/82] add check for calls length Signed-off-by: Nikita Karpukhin --- .../apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts index 8dbf06ed0f..072b155adc 100644 --- a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts +++ b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts @@ -152,7 +152,7 @@ describe('GoogleAnalytics', () => { }); }); - it('captures virtual pageviews', () => { + it('captures virtual pageviews instead of search events', () => { const config = new ConfigReader({ app: { analytics: { @@ -177,6 +177,7 @@ describe('GoogleAnalytics', () => { hitType: 'pageview', page: '/search?query=test search', }); + expect(ReactGA.testModeAPI.calls).toHaveLength(2); }); it('captures virtual pageviews alongside search events', () => { From 97070957add59dd9a6552f24b0956801aa599e5d Mon Sep 17 00:00:00 2001 From: Nikita Karpukhin Date: Wed, 30 Nov 2022 11:26:47 +0100 Subject: [PATCH 19/82] add support for search category Signed-off-by: Nikita Karpukhin --- plugins/analytics-module-ga/README.md | 10 ++++++---- plugins/analytics-module-ga/config.d.ts | 10 ++++++++-- .../AnalyticsApi/GoogleAnalytics.test.ts | 12 +++++++----- .../implementations/AnalyticsApi/GoogleAnalytics.ts | 10 ++++++++-- .../src/util/VirtualSearchPageView.ts | 6 ++++-- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/plugins/analytics-module-ga/README.md b/plugins/analytics-module-ga/README.md index 605044f37c..c2d79aa45d 100644 --- a/plugins/analytics-module-ga/README.md +++ b/plugins/analytics-module-ga/README.md @@ -181,7 +181,8 @@ app: virtualSearchPageView: mode: only # Defaults to 'disabled' mountPath: /virtual-search # Defaults to '/search' - queryParam: term # Defaults to 'query' + searchQuery: term # Defaults to 'query' + categoryQuery: sc # Omitted by default ``` Available `mode`s are: @@ -190,9 +191,10 @@ Available `mode`s are: - `only` - sends virtual pageviews _instead_ of `search` events - `both` - sends both virtual pageviews _and_ `search` events -Virtual pageviews will be sent to the path specified in the `mountPath`, and the search term will be -set as the value for query parameter `queryParam`, e.g. the example config above will result in -virtual pageviews being sent to `/virtual-search?term=SearchTermHere`. +Virtual pageviews will be sent to the path specified in the `mountPath`, the search term will be +set as the value for query parameter `searchQuery` and category (if provided) will be set as the value for +query parameter `categoryQuery`, e.g. the example config above will result in +virtual pageviews being sent to `/virtual-search?term=SearchTermHere&sc=CategoryHere`. ### Debugging and Testing diff --git a/plugins/analytics-module-ga/config.d.ts b/plugins/analytics-module-ga/config.d.ts index bf9c3b304b..188727bf93 100644 --- a/plugins/analytics-module-ga/config.d.ts +++ b/plugins/analytics-module-ga/config.d.ts @@ -72,11 +72,17 @@ export interface Config { */ mountPath?: string; /** - * Specifies which query param is used in the virtual pageview URL. + * Specifies which query param is used for the term query in the virtual pageview URL. * Defaults to `query`. * @visibility frontend */ - queryParam?: string; + searchQuery?: string; + /** + * Specifies which query param is used for the category query in the virtual pageview URL. + * Skipped by default. + * @visibility frontend + */ + categoryQuery?: string; }; /** diff --git a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts index 072b155adc..65e9256aa8 100644 --- a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts +++ b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.test.ts @@ -25,6 +25,7 @@ describe('GoogleAnalytics', () => { pluginId: 'some-plugin', routeRef: 'unknown', releaseNum: 1337, + searchTypes: 'test category', }; const trackingId = 'UA-000000-0'; const basicValidConfig = new ConfigReader({ @@ -175,7 +176,7 @@ describe('GoogleAnalytics', () => { expect(command).toBe('send'); expect(data).toMatchObject({ hitType: 'pageview', - page: '/search?query=test search', + page: '/search?query=test+search', }); expect(ReactGA.testModeAPI.calls).toHaveLength(2); }); @@ -203,7 +204,7 @@ describe('GoogleAnalytics', () => { expect(pageviewCommand).toBe('send'); expect(pageViewData).toMatchObject({ hitType: 'pageview', - page: '/search?query=test search', + page: '/search?query=test+search', }); const [searchCommand, searchData] = ReactGA.testModeAPI.calls[2]; expect(searchCommand).toBe('send'); @@ -215,7 +216,7 @@ describe('GoogleAnalytics', () => { }); }); - it('captures virtual pageviews on custom route with custom query param', () => { + it('captures virtual pageviews on custom route with custom search query and custom category', () => { const config = new ConfigReader({ app: { analytics: { @@ -225,7 +226,8 @@ describe('GoogleAnalytics', () => { virtualSearchPageView: { mode: 'only', mountPath: '/custom', - queryParam: 'term', + searchQuery: 'term', + categoryQuery: 'sc', }, }, }, @@ -242,7 +244,7 @@ describe('GoogleAnalytics', () => { expect(command).toBe('send'); expect(data).toMatchObject({ hitType: 'pageview', - page: '/custom?term=test search', + page: '/custom?term=test+search&sc=test+category', }); }); diff --git a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts index 16e4807008..17c5b26be1 100644 --- a/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts +++ b/plugins/analytics-module-ga/src/apis/implementations/AnalyticsApi/GoogleAnalytics.ts @@ -167,9 +167,15 @@ export class GoogleAnalytics implements AnalyticsApi { } if (this.virtualSearchPageView.mode !== 'disabled' && action === 'search') { - const { mountPath, queryParam } = this.virtualSearchPageView; + const { mountPath, searchQuery, categoryQuery } = + this.virtualSearchPageView; + const params = new URLSearchParams(); + params.set(searchQuery, subject); + if (categoryQuery) { + params.set(categoryQuery, context.searchTypes?.toString() ?? ''); + } this.capture.pageview( - `${mountPath}?${queryParam}=${subject}`, + `${mountPath}?${params.toString()}`, customMetadata, ); if (this.virtualSearchPageView.mode === 'only') { diff --git a/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts b/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts index 8da2eebca3..bb3d40afd3 100644 --- a/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts +++ b/plugins/analytics-module-ga/src/util/VirtualSearchPageView.ts @@ -20,7 +20,8 @@ type VirtualSearchPageViewType = 'disabled' | 'only' | 'both'; export type VirtualSearchPageViewConfig = { mode: VirtualSearchPageViewType; mountPath: string; - queryParam: string; + searchQuery: string; + categoryQuery?: string; }; function isVirtualSearchPageViewType( @@ -38,6 +39,7 @@ export function parseVirtualSearchPageViewConfig( ? vspvModeString : 'disabled', mountPath: config?.getOptionalString('mountPath') ?? '/search', - queryParam: config?.getOptionalString('queryParam') ?? 'query', + searchQuery: config?.getOptionalString('searchQuery') ?? 'query', + categoryQuery: config?.getOptionalString('categoryQuery'), }; } From 0675bb636d6b1987e0a8495e2b3e2315eb421233 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 12:26:37 +0000 Subject: [PATCH 20/82] Update dependency @swc/core to v1.3.21 Signed-off-by: Renovate Bot --- storybook/yarn.lock | 86 ++++++++++++++++++++++----------------------- yarn.lock | 86 ++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/storybook/yarn.lock b/storybook/yarn.lock index 29f94941ef..5867c2fb08 100644 --- a/storybook/yarn.lock +++ b/storybook/yarn.lock @@ -2966,90 +2966,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-darwin-arm64@npm:1.3.20" +"@swc/core-darwin-arm64@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-darwin-arm64@npm:1.3.21" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-darwin-x64@npm:1.3.20" +"@swc/core-darwin-x64@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-darwin-x64@npm:1.3.21" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.20" +"@swc/core-linux-arm-gnueabihf@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.21" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.20" +"@swc/core-linux-arm64-gnu@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.21" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.20" +"@swc/core-linux-arm64-musl@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.21" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.20" +"@swc/core-linux-x64-gnu@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.21" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-x64-musl@npm:1.3.20" +"@swc/core-linux-x64-musl@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-x64-musl@npm:1.3.21" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.20" +"@swc/core-win32-arm64-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.21" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.20" +"@swc/core-win32-ia32-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.21" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.20" +"@swc/core-win32-x64-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.21" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.20 - resolution: "@swc/core@npm:1.3.20" + version: 1.3.21 + resolution: "@swc/core@npm:1.3.21" dependencies: - "@swc/core-darwin-arm64": 1.3.20 - "@swc/core-darwin-x64": 1.3.20 - "@swc/core-linux-arm-gnueabihf": 1.3.20 - "@swc/core-linux-arm64-gnu": 1.3.20 - "@swc/core-linux-arm64-musl": 1.3.20 - "@swc/core-linux-x64-gnu": 1.3.20 - "@swc/core-linux-x64-musl": 1.3.20 - "@swc/core-win32-arm64-msvc": 1.3.20 - "@swc/core-win32-ia32-msvc": 1.3.20 - "@swc/core-win32-x64-msvc": 1.3.20 + "@swc/core-darwin-arm64": 1.3.21 + "@swc/core-darwin-x64": 1.3.21 + "@swc/core-linux-arm-gnueabihf": 1.3.21 + "@swc/core-linux-arm64-gnu": 1.3.21 + "@swc/core-linux-arm64-musl": 1.3.21 + "@swc/core-linux-x64-gnu": 1.3.21 + "@swc/core-linux-x64-musl": 1.3.21 + "@swc/core-win32-arm64-msvc": 1.3.21 + "@swc/core-win32-ia32-msvc": 1.3.21 + "@swc/core-win32-x64-msvc": 1.3.21 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -3073,7 +3073,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: 646c37e3521f04cd08061ab67a4388959e4b234c38eba2eb9fe0fd615dedb6ff9412264789e58af7c2d24b3e5a7ea456efd060e3760d7a91509234b5b5983ad7 + checksum: c66cd9320c595c68b87c8d90dc9a978099dd25a84c5e9795a8c7fec95fecdd8481da82076a828880a226ad2c0e57155c0a2b97768e99dee042a74182056bda46 languageName: node linkType: hard diff --git a/yarn.lock b/yarn.lock index 340f5edf7e..fd63e0dc7a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13121,90 +13121,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-darwin-arm64@npm:1.3.20" +"@swc/core-darwin-arm64@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-darwin-arm64@npm:1.3.21" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-darwin-x64@npm:1.3.20" +"@swc/core-darwin-x64@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-darwin-x64@npm:1.3.21" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.20" +"@swc/core-linux-arm-gnueabihf@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.21" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.20" +"@swc/core-linux-arm64-gnu@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.21" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.20" +"@swc/core-linux-arm64-musl@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.21" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.20" +"@swc/core-linux-x64-gnu@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.21" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-linux-x64-musl@npm:1.3.20" +"@swc/core-linux-x64-musl@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-linux-x64-musl@npm:1.3.21" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.20" +"@swc/core-win32-arm64-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.21" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.20" +"@swc/core-win32-ia32-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.21" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.20": - version: 1.3.20 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.20" +"@swc/core-win32-x64-msvc@npm:1.3.21": + version: 1.3.21 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.21" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.20 - resolution: "@swc/core@npm:1.3.20" + version: 1.3.21 + resolution: "@swc/core@npm:1.3.21" dependencies: - "@swc/core-darwin-arm64": 1.3.20 - "@swc/core-darwin-x64": 1.3.20 - "@swc/core-linux-arm-gnueabihf": 1.3.20 - "@swc/core-linux-arm64-gnu": 1.3.20 - "@swc/core-linux-arm64-musl": 1.3.20 - "@swc/core-linux-x64-gnu": 1.3.20 - "@swc/core-linux-x64-musl": 1.3.20 - "@swc/core-win32-arm64-msvc": 1.3.20 - "@swc/core-win32-ia32-msvc": 1.3.20 - "@swc/core-win32-x64-msvc": 1.3.20 + "@swc/core-darwin-arm64": 1.3.21 + "@swc/core-darwin-x64": 1.3.21 + "@swc/core-linux-arm-gnueabihf": 1.3.21 + "@swc/core-linux-arm64-gnu": 1.3.21 + "@swc/core-linux-arm64-musl": 1.3.21 + "@swc/core-linux-x64-gnu": 1.3.21 + "@swc/core-linux-x64-musl": 1.3.21 + "@swc/core-win32-arm64-msvc": 1.3.21 + "@swc/core-win32-ia32-msvc": 1.3.21 + "@swc/core-win32-x64-msvc": 1.3.21 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -13228,7 +13228,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: 646c37e3521f04cd08061ab67a4388959e4b234c38eba2eb9fe0fd615dedb6ff9412264789e58af7c2d24b3e5a7ea456efd060e3760d7a91509234b5b5983ad7 + checksum: c66cd9320c595c68b87c8d90dc9a978099dd25a84c5e9795a8c7fec95fecdd8481da82076a828880a226ad2c0e57155c0a2b97768e99dee042a74182056bda46 languageName: node linkType: hard From 462c1d012e8e35bf86daefbe6f8850b4ea1b974f Mon Sep 17 00:00:00 2001 From: Luca Huettner Date: Wed, 30 Nov 2022 15:11:25 +0100 Subject: [PATCH 21/82] refactor: Remove the header from the default catalog page Signed-off-by: Luca Huettner --- .changeset/modern-camels-cheat.md | 5 ++++ .../CatalogKindHeader/CatalogKindHeader.tsx | 5 +++- .../CatalogPage/DefaultCatalogPage.tsx | 25 ++++++++----------- 3 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 .changeset/modern-camels-cheat.md diff --git a/.changeset/modern-camels-cheat.md b/.changeset/modern-camels-cheat.md new file mode 100644 index 0000000000..69c680b989 --- /dev/null +++ b/.changeset/modern-camels-cheat.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Remove `CatalogKindHeader` from `DefaultCatalogPage`. Deprecate `CatalogKindHeader`. diff --git a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx index bb9e98d9c3..2cb8709c34 100644 --- a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx +++ b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx @@ -59,7 +59,10 @@ export interface CatalogKindHeaderProps { initialFilter?: string; } -/** @public */ +/** + * @public + * @deprecated + */ export function CatalogKindHeader(props: CatalogKindHeaderProps) { const { initialFilter = 'component', allowedKinds } = props; const classes = useStyles(); diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.tsx index c5208c94f5..644453f5fb 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.tsx @@ -39,7 +39,6 @@ import { import React, { ReactNode } from 'react'; import { createComponentRouteRef } from '../../routes'; import { CatalogTable, CatalogTableRow } from '../CatalogTable'; -import { CatalogKindHeader } from '../CatalogKindHeader'; import { useCatalogPluginOptions } from '../../options'; /** @@ -73,17 +72,15 @@ export function DefaultCatalogPage(props: DefaultCatalogPageProps) { return ( - - - } - > - - All your software catalog entities - + + + + All your software catalog entities + + @@ -103,8 +100,8 @@ export function DefaultCatalogPage(props: DefaultCatalogPageProps) { /> - - + + ); } From 45eb4d23cf04addaf21364bcb4a4615c27fe9d48 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Wed, 30 Nov 2022 15:27:08 +0100 Subject: [PATCH 22/82] Be persistent in attempting to clean up indices on error Signed-off-by: Eric Peterson --- .changeset/search-lieutenant-dangle.md | 5 ++ .../engines/ElasticSearchSearchEngine.test.ts | 30 +++++++----- .../src/engines/ElasticSearchSearchEngine.ts | 46 ++++++++++++++----- 3 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 .changeset/search-lieutenant-dangle.md diff --git a/.changeset/search-lieutenant-dangle.md b/.changeset/search-lieutenant-dangle.md new file mode 100644 index 0000000000..75f1bca915 --- /dev/null +++ b/.changeset/search-lieutenant-dangle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Fixed a bug that prevented indices from being cleaned up under some circumstances, which could have led to shard exhaustion. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 3a237664a8..602a158b98 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -68,6 +68,14 @@ const customIndexTemplate = { }, }; +const advanceTimersByNTimes = async (n = 1, time = 1000) => { + for (let i = 0; i < n; i++) { + await Promise.resolve(); + jest.advanceTimersByTime(time); + await Promise.resolve(); + } +}; + describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; @@ -855,35 +863,33 @@ describe('ElasticSearchSearchEngine', () => { }); it('should check for and delete expected index', async () => { - const existsSpy = jest.fn().mockReturnValue('truthy value'); const deleteSpy = jest.fn().mockReturnValue({}); - mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); await errorHandler(error); // Check and delete HTTP requests were made. - expect(existsSpy).toHaveBeenCalled(); expect(deleteSpy).toHaveBeenCalled(); }); - it('should not delete index if none exists', async () => { - // Exists call returns 404 on no index. - const existsSpy = jest.fn().mockReturnValue( + it('should retry delete index up to 5 times', async () => { + // Delete call returns 404 + const deleteSpy = jest.fn().mockReturnValue( new errors.ResponseError({ statusCode: 404, body: { status: 404 }, } as unknown as any), ); - const deleteSpy = jest.fn().mockReturnValue({}); - mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); - await errorHandler(error); + // Call the error handler and advance timers + jest.useFakeTimers(); + errorHandler(error); + await advanceTimersByNTimes(10); + jest.useRealTimers(); - // Check request was made, but no delete request was made. - expect(existsSpy).toHaveBeenCalled(); - expect(deleteSpy).not.toHaveBeenCalled(); + // Check request was made 5 times + expect(deleteSpy).toHaveBeenCalledTimes(5); }); }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 72bab5b3b6..f3db46956d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -266,19 +266,41 @@ export class ElasticSearchSearchEngine implements SearchEngine { // Attempt cleanup upon failure. indexer.on('error', async e => { this.logger.error(`Failed to index documents for type ${type}`, e); - try { - const response = await this.elasticSearchClientWrapper.indexExists({ - index: indexer.indexName, - }); - const indexCreated = response.body; - if (indexCreated) { - this.logger.info(`Removing created index ${indexer.indexName}`); - await this.elasticSearchClientWrapper.deleteIndex({ - index: indexer.indexName, - }); + let cleanupError: Error | undefined; + + // In some cases, a failure may have occurred before the indexer was able + // to complete initialization. Try up to 5 times to remove the dangling + // index. + await new Promise(async done => { + const maxAttempts = 5; + let attempts = 0; + + while (attempts < maxAttempts) { + try { + await this.elasticSearchClientWrapper.deleteIndex({ + index: indexer.indexName, + }); + + attempts = maxAttempts; + cleanupError = undefined; + done(); + } catch (err) { + cleanupError = err; + } + + // Wait 1 second between retries. + await new Promise(okay => setTimeout(okay, 1000)); + + attempts++; } - } catch (error) { - this.logger.error(`Unable to clean up elastic index: ${error}`); + }); + + if (cleanupError) { + this.logger.error( + `Unable to clean up elastic index ${indexer.indexName}: ${cleanupError}`, + ); + } else { + this.logger.info(`Removed partial, failed index ${indexer.indexName}`); } }); From 38f9551faf433fe9e93404eeea32cfac28ea77d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 15:05:25 +0000 Subject: [PATCH 23/82] Update dependency yeoman-environment to v3.13.0 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index fe21e797f9..441430dd19 100644 --- a/yarn.lock +++ b/yarn.lock @@ -38654,8 +38654,8 @@ __metadata: linkType: hard "yeoman-environment@npm:^3.9.1": - version: 3.12.1 - resolution: "yeoman-environment@npm:3.12.1" + version: 3.13.0 + resolution: "yeoman-environment@npm:3.13.0" dependencies: "@npmcli/arborist": ^4.0.4 are-we-there-yet: ^2.0.0 @@ -38698,7 +38698,7 @@ __metadata: mem-fs-editor: ^8.1.2 || ^9.0.0 bin: yoe: cli/index.js - checksum: 71e777fcfa4baf26f9848265292447d7283f4bcb0e6b781018c8f8610c47c430e5a331eb0da03491621a7bcdd6ed40a2854c664a4f9f1ceb9ee111def37f52d1 + checksum: 2d622d18d2e3fff179477b6dabbdee8b380d58b5d639d300c3ace75e0533223cecb9b325baaf0f46f71879eaa5778d64b11fa660403a26ec3086b2a428484ce6 languageName: node linkType: hard From 2e6d0a9558bb54475468f32912c2d9e46b3d8bb3 Mon Sep 17 00:00:00 2001 From: Luca Huettner Date: Wed, 30 Nov 2022 16:17:38 +0100 Subject: [PATCH 24/82] chore: Update api report Signed-off-by: Luca Huettner --- plugins/catalog/api-report.md | 2 +- .../src/components/CatalogKindHeader/CatalogKindHeader.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog/api-report.md b/plugins/catalog/api-report.md index 3f72df00e2..74f64882a8 100644 --- a/plugins/catalog/api-report.md +++ b/plugins/catalog/api-report.md @@ -73,7 +73,7 @@ export const CatalogEntityPage: () => JSX.Element; // @public (undocumented) export const CatalogIndexPage: (props: DefaultCatalogPageProps) => JSX.Element; -// @public (undocumented) +// @public @deprecated (undocumented) export function CatalogKindHeader(props: CatalogKindHeaderProps): JSX.Element; // @public diff --git a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx index 2cb8709c34..61a3c6fb4f 100644 --- a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx +++ b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.tsx @@ -61,7 +61,7 @@ export interface CatalogKindHeaderProps { /** * @public - * @deprecated + * @deprecated Might be removed in a future release. */ export function CatalogKindHeader(props: CatalogKindHeaderProps) { const { initialFilter = 'component', allowedKinds } = props; From 16b7c2fccda14a1de93d6c94de80be999513d37c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 18:33:59 +0000 Subject: [PATCH 25/82] Update dependency @rollup/plugin-yaml to v4 Signed-off-by: Renovate Bot --- .changeset/renovate-bf8dfb3.md | 5 +++++ packages/cli/package.json | 2 +- yarn.lock | 31 +++++++++++++++++-------------- 3 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 .changeset/renovate-bf8dfb3.md diff --git a/.changeset/renovate-bf8dfb3.md b/.changeset/renovate-bf8dfb3.md new file mode 100644 index 0000000000..57cabf075b --- /dev/null +++ b/.changeset/renovate-bf8dfb3.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Updated dependency `@rollup/plugin-yaml` to `^4.0.0`. diff --git a/packages/cli/package.json b/packages/cli/package.json index eef838d8ca..ca2bdfb56c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -42,7 +42,7 @@ "@rollup/plugin-commonjs": "^23.0.0", "@rollup/plugin-json": "^5.0.0", "@rollup/plugin-node-resolve": "^13.0.6", - "@rollup/plugin-yaml": "^3.1.0", + "@rollup/plugin-yaml": "^4.0.0", "@spotify/eslint-config-base": "^14.0.0", "@spotify/eslint-config-react": "^14.0.0", "@spotify/eslint-config-typescript": "^14.0.0", diff --git a/yarn.lock b/yarn.lock index 441430dd19..b38d765bd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3613,7 +3613,7 @@ __metadata: "@rollup/plugin-commonjs": ^23.0.0 "@rollup/plugin-json": ^5.0.0 "@rollup/plugin-node-resolve": ^13.0.6 - "@rollup/plugin-yaml": ^3.1.0 + "@rollup/plugin-yaml": ^4.0.0 "@spotify/eslint-config-base": ^14.0.0 "@spotify/eslint-config-react": ^14.0.0 "@spotify/eslint-config-typescript": ^14.0.0 @@ -12683,16 +12683,19 @@ __metadata: languageName: node linkType: hard -"@rollup/plugin-yaml@npm:^3.1.0": - version: 3.1.0 - resolution: "@rollup/plugin-yaml@npm:3.1.0" +"@rollup/plugin-yaml@npm:^4.0.0": + version: 4.0.1 + resolution: "@rollup/plugin-yaml@npm:4.0.1" dependencies: - "@rollup/pluginutils": ^3.1.0 - js-yaml: ^3.14.0 - tosource: ^1.0.0 + "@rollup/pluginutils": ^5.0.1 + js-yaml: ^4.1.0 + tosource: ^2.0.0-alpha.3 peerDependencies: - rollup: ^1.20.0 || ^2.0.0 - checksum: be99aa097c480a24e6bedf85de2f8e21e287baf443155e7f2e765c939dda53f63cda674476f170b0397e3c69ce3bc825f0ad4041ac6fe25912480c27423e1b37 + rollup: ^1.20.0||^2.0.0||^3.0.0 + peerDependenciesMeta: + rollup: + optional: true + checksum: d48e248a9acd3110c889d745562e03e5a16e5861838c6b6d92701e59c0d4a24af374ee7bd81db0ab8e6d021cdb925c05bd7d30149e4e9a0fce4d899170a78857 languageName: node linkType: hard @@ -26263,7 +26266,7 @@ __metadata: languageName: node linkType: hard -"js-yaml@npm:^3.10.0, js-yaml@npm:^3.13.0, js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.0, js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1, js-yaml@npm:^3.8.3": +"js-yaml@npm:^3.10.0, js-yaml@npm:^3.13.0, js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1, js-yaml@npm:^3.8.3": version: 3.14.1 resolution: "js-yaml@npm:3.14.1" dependencies: @@ -36450,10 +36453,10 @@ __metadata: languageName: node linkType: hard -"tosource@npm:^1.0.0": - version: 1.0.0 - resolution: "tosource@npm:1.0.0" - checksum: 683fc64700484cd749b6eed461c2d105e251df7caf886484ff824bcf1249e365210d65c4f2ca3ced30baa34ea7912c656984e084aaada669920b44e2437c1c05 +"tosource@npm:^2.0.0-alpha.3": + version: 2.0.0-alpha.3 + resolution: "tosource@npm:2.0.0-alpha.3" + checksum: bc03a7571de8ed4306e6721283fa891f2adcab9dd80c46f6f177d4259b34bb192fe3a2cb3e1e2ce16f9db0bc7e534acfcb5478ab094b0ba255f98abfce6dab46 languageName: node linkType: hard From 830687539f5f12bb1d7b124ff66c678c7e1e8f85 Mon Sep 17 00:00:00 2001 From: Carlos Esteban Lopez Date: Wed, 16 Nov 2022 14:12:38 -0500 Subject: [PATCH 26/82] feat: Apply component guidelines for @backstage/core-components for src/components Signed-off-by: Carlos Esteban Lopez --- .changeset/strong-peaches-melt.md | 5 +++ .../components/AlertDisplay/AlertDisplay.tsx | 33 +++++++------- .../src/components/Avatar/Avatar.tsx | 24 +++++++++-- .../components/CodeSnippet/CodeSnippet.tsx | 21 ++++----- .../DismissableBanner/DismissableBanner.tsx | 2 +- .../MissingAnnotationEmptyState.tsx | 17 ++++---- .../FeatureCalloutCircular.tsx | 21 ++++----- .../HeaderIconLinkRow/IconLinkVertical.tsx | 17 +++++--- .../HorizontalScrollGrid.tsx | 10 ++--- .../src/components/Lifecycle/Lifecycle.tsx | 15 ++++--- .../src/components/Link/Link.tsx | 13 +++--- .../components/LogViewer/RealLogViewer.tsx | 23 +++++----- .../src/components/Progress/Progress.tsx | 16 ++++--- .../src/components/ProgressBars/Gauge.tsx | 15 ++++--- .../src/components/ProgressBars/GaugeCard.tsx | 7 +-- .../components/ProgressBars/LinearGauge.tsx | 13 +++--- .../src/components/Select/Select.tsx | 23 +++++----- .../Select/static/ClosedDropdown.tsx | 4 +- .../Select/static/OpenedDropdown.tsx | 4 +- .../SimpleStepper/SimpleStepperFooter.tsx | 12 +++--- .../SimpleStepper/SimpleStepperStep.tsx | 10 +++-- .../src/components/Status/Status.tsx | 26 ++++++----- .../StructuredMetadataTable/MetadataTable.tsx | 20 +++++---- .../StructuredMetadataTable.tsx | 11 ++++- .../src/components/Table/Filters.tsx | 23 +++++----- .../src/components/Table/SubvalueCell.tsx | 10 ++--- .../src/components/Table/Table.tsx | 43 ++++++++++--------- .../src/components/Tabs/TabBar.tsx | 2 +- .../src/components/Tabs/TabIcon.tsx | 6 +-- .../src/components/Tabs/TabPanel.tsx | 7 ++- .../src/components/Tabs/Tabs.tsx | 13 +++--- .../components/WarningPanel/WarningPanel.tsx | 2 +- .../src/layout/ErrorPage/ErrorPage.tsx | 2 +- .../src/layout/Sidebar/SidebarSubmenu.tsx | 2 +- .../src/layout/Sidebar/SidebarSubmenuItem.tsx | 4 +- 35 files changed, 271 insertions(+), 205 deletions(-) create mode 100644 .changeset/strong-peaches-melt.md diff --git a/.changeset/strong-peaches-melt.md b/.changeset/strong-peaches-melt.md new file mode 100644 index 0000000000..d866b8ba54 --- /dev/null +++ b/.changeset/strong-peaches-melt.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Sync components in @backstage/core-components with the Component Design Guidelines diff --git a/packages/core-components/src/components/AlertDisplay/AlertDisplay.tsx b/packages/core-components/src/components/AlertDisplay/AlertDisplay.tsx index c1f8f51db0..83c11647bb 100644 --- a/packages/core-components/src/components/AlertDisplay/AlertDisplay.tsx +++ b/packages/core-components/src/components/AlertDisplay/AlertDisplay.tsx @@ -13,14 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React, { useEffect, useState } from 'react'; -import Snackbar from '@material-ui/core/Snackbar'; +import { alertApiRef, AlertMessage, useApi } from '@backstage/core-plugin-api'; import IconButton from '@material-ui/core/IconButton'; +import Snackbar from '@material-ui/core/Snackbar'; +import Typography from '@material-ui/core/Typography'; import CloseIcon from '@material-ui/icons/Close'; import { Alert } from '@material-ui/lab'; -import { AlertMessage, useApi, alertApiRef } from '@backstage/core-plugin-api'; import pluralize from 'pluralize'; +import React, { useEffect, useState } from 'react'; + +// TODO: improve on this and promote to a shared component for use by all apps. + +/** @public */ +export type AlertDisplayProps = { + anchorOrigin?: { + vertical: 'top' | 'bottom'; + horizontal: 'left' | 'center' | 'right'; + }; +}; /** * Displays alerts from {@link @backstage/core-plugin-api#AlertApi} @@ -30,17 +40,6 @@ import pluralize from 'pluralize'; * * Shown as SnackBar at the center top of the page by default. Configurable with props. */ - -// TODO: improve on this and promote to a shared component for use by all apps. - -export type AlertDisplayProps = { - anchorOrigin?: { - vertical: 'top' | 'bottom'; - horizontal: 'left' | 'center' | 'right'; - }; -}; - -/** @public */ export function AlertDisplay(props: AlertDisplayProps) { const [messages, setMessages] = useState>([]); const alertApi = useApi(alertApiRef); @@ -82,7 +81,7 @@ export function AlertDisplay(props: AlertDisplayProps) { } severity={firstMessage.severity} > - + {String(firstMessage.message)} {messages.length > 1 && ( {` (${messages.length - 1} older ${pluralize( @@ -90,7 +89,7 @@ export function AlertDisplay(props: AlertDisplayProps) { messages.length - 1, )})`} )} - + ); diff --git a/packages/core-components/src/components/Avatar/Avatar.tsx b/packages/core-components/src/components/Avatar/Avatar.tsx index af8dabf554..950f57668e 100644 --- a/packages/core-components/src/components/Avatar/Avatar.tsx +++ b/packages/core-components/src/components/Avatar/Avatar.tsx @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { CSSProperties } from 'react'; -import { createStyles, makeStyles, Theme } from '@material-ui/core/styles'; import MaterialAvatar from '@material-ui/core/Avatar'; +import { createStyles, makeStyles, Theme } from '@material-ui/core/styles'; +import Typography from '@material-ui/core/Typography'; +import React, { CSSProperties } from 'react'; + import { extractInitials, stringToColor } from './utils'; /** @public */ @@ -28,6 +30,8 @@ const useStyles = makeStyles( width: '4rem', height: '4rem', color: '#fff', + }, + avatarText: { fontWeight: theme.typography.fontWeightBold, letterSpacing: '1px', textTransform: 'uppercase', @@ -68,6 +72,11 @@ export function Avatar(props: AvatarProps) { const { displayName, picture, customStyles } = props; const classes = useStyles(); let styles = { ...customStyles }; + const fontStyles = { + fontFamily: styles.fontFamily, + fontSize: styles.fontSize, + fontWeight: styles.fontWeight, + }; // We only calculate the background color if there's not an avatar // picture. If there is a picture, it might have a transparent // background and we don't know whether the calculated background @@ -85,7 +94,16 @@ export function Avatar(props: AvatarProps) { className={classes.avatar} style={styles} > - {displayName && extractInitials(displayName)} + {displayName && ( + + {extractInitials(displayName)} + + )} ); } diff --git a/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx b/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx index fd83797a88..cd6ad07a5c 100644 --- a/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx +++ b/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React from 'react'; -import { useTheme } from '@material-ui/core/styles'; import { BackstageTheme } from '@backstage/theme'; -import { CopyTextButton } from '../CopyTextButton'; -import type {} from 'react-syntax-highlighter'; -import { default as LightAsync } from 'react-syntax-highlighter/dist/esm/light-async'; +import Box from '@material-ui/core/Box'; +import { useTheme } from '@material-ui/core/styles'; +import React from 'react'; +import LightAsync from 'react-syntax-highlighter/dist/esm/light-async'; import dark from 'react-syntax-highlighter/dist/esm/styles/hljs/dark'; import docco from 'react-syntax-highlighter/dist/esm/styles/hljs/docco'; +import { CopyTextButton } from '../CopyTextButton'; + +import type {} from 'react-syntax-highlighter'; /** * Properties for {@link CodeSnippet} * @@ -87,7 +88,7 @@ export function CodeSnippet(props: CodeSnippetProps) { const highlightColor = theme.palette.type === 'dark' ? '#256bf3' : '#e6ffed'; return ( -
+ {showCopyCodeButton && ( -
+ -
+
)} -
+ ); } diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 99113fcdb1..c18f9b605f 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -58,7 +58,7 @@ const useStyles = makeStyles( zIndex: 'unset', }, icon: { - fontSize: 20, + fontSize: theme.typography.h6.fontSize, }, content: { width: '100%', diff --git a/packages/core-components/src/components/EmptyState/MissingAnnotationEmptyState.tsx b/packages/core-components/src/components/EmptyState/MissingAnnotationEmptyState.tsx index 59596e6d2b..1910d6e7fd 100644 --- a/packages/core-components/src/components/EmptyState/MissingAnnotationEmptyState.tsx +++ b/packages/core-components/src/components/EmptyState/MissingAnnotationEmptyState.tsx @@ -13,15 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React from 'react'; -import { makeStyles } from '@material-ui/core/styles'; -import Button from '@material-ui/core/Button'; -import Typography from '@material-ui/core/Typography'; import { BackstageTheme } from '@backstage/theme'; +import Box from '@material-ui/core/Box'; +import Button from '@material-ui/core/Button'; +import { makeStyles } from '@material-ui/core/styles'; +import Typography from '@material-ui/core/Typography'; +import React from 'react'; + +import { CodeSnippet } from '../CodeSnippet'; import { Link } from '../Link'; import { EmptyState } from './EmptyState'; -import { CodeSnippet } from '../CodeSnippet'; const COMPONENT_YAML_TEMPLATE = `apiVersion: backstage.io/v1alpha1 kind: Component @@ -109,7 +110,7 @@ export function MissingAnnotationEmptyState(props: Props) { Add the annotation to your component YAML as shown in the highlighted example below: -
+ -
+ diff --git a/packages/core-components/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx b/packages/core-components/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx index cac3c8166b..b967ee0a9f 100644 --- a/packages/core-components/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx +++ b/packages/core-components/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import Box from '@material-ui/core/Box'; import ClickAwayListener from '@material-ui/core/ClickAwayListener'; import { makeStyles } from '@material-ui/core/styles'; import Typography from '@material-ui/core/Typography'; @@ -26,6 +26,7 @@ import React, { useState, } from 'react'; import { createPortal } from 'react-dom'; + import { usePortal } from './lib/usePortal'; import { useShowCallout } from './lib/useShowCallout'; @@ -168,14 +169,14 @@ export function FeatureCalloutCircular(props: PropsWithChildren) { return ( <> -
+ {children} -
+ {createPortal( -
+ <> -
) { role="button" tabIndex={0} > -
-
-
+ + ) { {title} {description} -
+ -
, +
, portalElement, )} diff --git a/packages/core-components/src/components/HeaderIconLinkRow/IconLinkVertical.tsx b/packages/core-components/src/components/HeaderIconLinkRow/IconLinkVertical.tsx index f128455549..df03c0f3c6 100644 --- a/packages/core-components/src/components/HeaderIconLinkRow/IconLinkVertical.tsx +++ b/packages/core-components/src/components/HeaderIconLinkRow/IconLinkVertical.tsx @@ -18,6 +18,8 @@ import classnames from 'classnames'; import { makeStyles } from '@material-ui/core/styles'; import LinkIcon from '@material-ui/icons/Link'; import { Link } from '../Link'; +import Box from '@material-ui/core/Box'; +import Typography from '@material-ui/core/Typography'; export type IconLinkVerticalProps = { color?: 'primary' | 'secondary'; @@ -56,9 +58,8 @@ const useIconStyles = makeStyles( color: theme.palette.secondary.main, }, label: { - fontSize: '0.7rem', textTransform: 'uppercase', - fontWeight: 600, + fontWeight: theme.typography.fontWeightBold, letterSpacing: 1.2, }, }), @@ -79,10 +80,12 @@ export function IconLinkVertical({ if (disabled) { return ( -
+ {icon} - {label} -
+ + {label} + + ); } @@ -94,7 +97,9 @@ export function IconLinkVertical({ onClick={onClick} > {icon} - {label} + + {label} + ); } diff --git a/packages/core-components/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx b/packages/core-components/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx index 4ab49596fe..c4c9dadb78 100644 --- a/packages/core-components/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx +++ b/packages/core-components/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import Box from '@material-ui/core/Box'; import Grid from '@material-ui/core/Grid'; import IconButton from '@material-ui/core/IconButton'; import { makeStyles, Theme } from '@material-ui/core/styles'; @@ -227,7 +227,7 @@ export function HorizontalScrollGrid(props: PropsWithChildren) { }; return ( -
+ ) { > {children} -
-
) { )} -
+ ); } diff --git a/packages/core-components/src/components/Lifecycle/Lifecycle.tsx b/packages/core-components/src/components/Lifecycle/Lifecycle.tsx index f553129927..f0bbeaa7bd 100644 --- a/packages/core-components/src/components/Lifecycle/Lifecycle.tsx +++ b/packages/core-components/src/components/Lifecycle/Lifecycle.tsx @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React from 'react'; -import CSS from 'csstype'; import { makeStyles } from '@material-ui/core/styles'; +import Typography from '@material-ui/core/Typography'; +import CSS from 'csstype'; +import React from 'react'; type Props = CSS.Properties & { shorthand?: boolean; @@ -47,15 +47,16 @@ export function Lifecycle(props: Props) { const classes = useStyles(props); const { shorthand, alpha } = props; return shorthand ? ( - {alpha ? <>α : <>β} - + ) : ( - + {alpha ? 'Alpha' : 'Beta'} - + ); } diff --git a/packages/core-components/src/components/Link/Link.tsx b/packages/core-components/src/components/Link/Link.tsx index dfa38be44e..440fcb811f 100644 --- a/packages/core-components/src/components/Link/Link.tsx +++ b/packages/core-components/src/components/Link/Link.tsx @@ -13,21 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import { configApiRef, useAnalytics, useApi } from '@backstage/core-plugin-api'; -import classnames from 'classnames'; // eslint-disable-next-line no-restricted-imports import MaterialLink, { LinkProps as MaterialLinkProps, } from '@material-ui/core/Link'; import { makeStyles } from '@material-ui/core/styles'; +import Typography from '@material-ui/core/Typography'; +import classnames from 'classnames'; +import { trimEnd } from 'lodash'; import React, { ElementType } from 'react'; import { + createRoutesFromChildren, Link as RouterLink, LinkProps as RouterLinkProps, + Route, } from 'react-router-dom'; -import { trimEnd } from 'lodash'; -import { createRoutesFromChildren, Route } from 'react-router-dom'; export function isReactRouterBeta(): boolean { const [obj] = createRoutesFromChildren(} />); @@ -161,7 +162,9 @@ export const Link = React.forwardRef( className={classnames(classes.externalLink, props.className)} > {props.children} - , Opens in a new window + + , Opens in a new window + ) : ( // Interact with React Router for internal links diff --git a/packages/core-components/src/components/LogViewer/RealLogViewer.tsx b/packages/core-components/src/components/LogViewer/RealLogViewer.tsx index ac66d7cdf5..59e0bea44b 100644 --- a/packages/core-components/src/components/LogViewer/RealLogViewer.tsx +++ b/packages/core-components/src/components/LogViewer/RealLogViewer.tsx @@ -13,18 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React, { useEffect, useMemo, useRef } from 'react'; -import { useLocation } from 'react-router-dom'; +import Box from '@material-ui/core/Box'; import IconButton from '@material-ui/core/IconButton'; import CopyIcon from '@material-ui/icons/FileCopy'; +import classnames from 'classnames'; +import React, { useEffect, useMemo, useRef } from 'react'; +import { useLocation } from 'react-router-dom'; import AutoSizer from 'react-virtualized-auto-sizer'; import { FixedSizeList } from 'react-window'; + import { AnsiProcessor } from './AnsiProcessor'; -import { HEADER_SIZE, useStyles } from './styles'; -import classnames from 'classnames'; import { LogLine } from './LogLine'; import { LogViewerControls } from './LogViewerControls'; +import { HEADER_SIZE, useStyles } from './styles'; import { useLogViewerSearch } from './useLogViewerSearch'; import { useLogViewerSelection } from './useLogViewerSelection'; @@ -69,10 +70,10 @@ export function RealLogViewer(props: RealLogViewerProps) { return ( {({ height, width }) => ( -
-
+ + -
+ -
+ ); }} -
+
)} ); diff --git a/packages/core-components/src/components/Progress/Progress.tsx b/packages/core-components/src/components/Progress/Progress.tsx index f199c5640f..eb17b9eb25 100644 --- a/packages/core-components/src/components/Progress/Progress.tsx +++ b/packages/core-components/src/components/Progress/Progress.tsx @@ -13,23 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React, { useState, useEffect, PropsWithChildren } from 'react'; +import { BackstageTheme } from '@backstage/theme'; +import Box from '@material-ui/core/Box'; import LinearProgress, { LinearProgressProps, } from '@material-ui/core/LinearProgress'; +import { useTheme } from '@material-ui/core/styles'; +import React, { PropsWithChildren, useEffect, useState } from 'react'; export function Progress(props: PropsWithChildren) { + const theme = useTheme(); const [isVisible, setIsVisible] = useState(false); useEffect(() => { - const handle = setTimeout(() => setIsVisible(true), 250); + const handle = setTimeout( + () => setIsVisible(true), + theme.transitions.duration.short, + ); return () => clearTimeout(handle); - }, []); + }, [theme.transitions.duration.short]); return isVisible ? ( ) : ( -
+ ); } diff --git a/packages/core-components/src/components/ProgressBars/Gauge.tsx b/packages/core-components/src/components/ProgressBars/Gauge.tsx index 11f977b387..ca84a5f480 100644 --- a/packages/core-components/src/components/ProgressBars/Gauge.tsx +++ b/packages/core-components/src/components/ProgressBars/Gauge.tsx @@ -18,6 +18,7 @@ import { BackstagePalette, BackstageTheme } from '@backstage/theme'; import { makeStyles, useTheme } from '@material-ui/core/styles'; import { Circle } from 'rc-progress'; import React, { ReactNode, useEffect, useState } from 'react'; +import Box from '@material-ui/core/Box'; /** @public */ export type GaugeClassKey = @@ -38,8 +39,8 @@ const useStyles = makeStyles( top: '50%', left: '50%', transform: 'translate(-50%, -60%)', - fontSize: 45, - fontWeight: 'bold', + fontSize: theme.typography.pxToRem(45), + fontWeight: theme.typography.fontWeightBold, color: theme.palette.textContrast, }, description: { @@ -152,7 +153,7 @@ export function Gauge(props: GaugeProps) { }, [description, hoverRef]); return ( -
+ {description && isHovering ? ( -
{description}
+ {description} ) : ( -
+ {isNaN(value) ? 'N/A' : `${asActual}${unit}`} -
+
)} -
+
); } diff --git a/packages/core-components/src/components/ProgressBars/GaugeCard.tsx b/packages/core-components/src/components/ProgressBars/GaugeCard.tsx index 1114890523..3442395380 100644 --- a/packages/core-components/src/components/ProgressBars/GaugeCard.tsx +++ b/packages/core-components/src/components/ProgressBars/GaugeCard.tsx @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import Box from '@material-ui/core/Box'; import { makeStyles } from '@material-ui/core/styles'; import React, { ReactNode } from 'react'; + import { BottomLinkProps } from '../../layout/BottomLink'; import { InfoCard, InfoCardVariants } from '../../layout/InfoCard'; import { Gauge, GaugePropsGetColor } from './Gauge'; @@ -74,7 +75,7 @@ export function GaugeCard(props: Props) { }; return ( -
+ -
+ ); } diff --git a/packages/core-components/src/components/ProgressBars/LinearGauge.tsx b/packages/core-components/src/components/ProgressBars/LinearGauge.tsx index 643c3ec2ac..5bfe1b7a61 100644 --- a/packages/core-components/src/components/ProgressBars/LinearGauge.tsx +++ b/packages/core-components/src/components/ProgressBars/LinearGauge.tsx @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import React from 'react'; +import { BackstageTheme } from '@backstage/theme'; import { useTheme } from '@material-ui/core/styles'; import Tooltip from '@material-ui/core/Tooltip'; +import Typography from '@material-ui/core/Typography'; import { Line } from 'rc-progress'; -import { BackstageTheme } from '@backstage/theme'; -import { getProgressColor, GaugePropsGetColor } from './Gauge'; +import React from 'react'; + +import { GaugePropsGetColor, getProgressColor } from './Gauge'; type Props = { /** @@ -47,14 +48,14 @@ export function LinearGauge(props: Props) { }); return ( - + - + ); } diff --git a/packages/core-components/src/components/Select/Select.tsx b/packages/core-components/src/components/Select/Select.tsx index 7229f89653..9cb8822a05 100644 --- a/packages/core-components/src/components/Select/Select.tsx +++ b/packages/core-components/src/components/Select/Select.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import Box from '@material-ui/core/Box'; import Checkbox from '@material-ui/core/Checkbox'; import Chip from '@material-ui/core/Chip'; import ClickAwayListener from '@material-ui/core/ClickAwayListener'; @@ -30,6 +30,7 @@ import { } from '@material-ui/core/styles'; import Typography from '@material-ui/core/Typography'; import React, { useEffect, useState } from 'react'; + import ClosedDropdown from './static/ClosedDropdown'; import OpenedDropdown from './static/OpenedDropdown'; @@ -45,17 +46,17 @@ const BootstrapInput = withStyles( }, }, input: { - borderRadius: 4, + borderRadius: theme.shape.borderRadius, position: 'relative', backgroundColor: theme.palette.background.paper, border: '1px solid #ced4da', - fontSize: 16, - padding: '10px 26px 10px 12px', + fontSize: theme.typography.body1.fontSize, + padding: theme.spacing(1.25, 3.25, 1.25, 1.5), transition: theme.transitions.create(['border-color', 'box-shadow']), fontFamily: 'Helvetica Neue', '&:focus': { background: theme.palette.background.paper, - borderRadius: 4, + borderRadius: theme.shape.borderRadius, }, }, }), @@ -81,7 +82,7 @@ const useStyles = makeStyles( label: { transform: 'initial', fontWeight: 'bold', - fontSize: 14, + fontSize: theme.typography.body2.fontSize, fontFamily: theme.typography.fontFamily, color: theme.palette.text.primary, '&.Mui-focused': { @@ -91,7 +92,7 @@ const useStyles = makeStyles( formLabel: { transform: 'initial', fontWeight: 'bold', - fontSize: 14, + fontSize: theme.typography.body2.fontSize, fontFamily: theme.typography.fontFamily, color: theme.palette.text.primary, '&.Mui-focused': { @@ -196,7 +197,7 @@ export function SelectComponent(props: SelectProps) { }; return ( -
+ {label} @@ -217,7 +218,7 @@ export function SelectComponent(props: SelectProps) { tabIndex={0} renderValue={s => multiple && (value as any[]).length !== 0 ? ( -
+ {(s as string[]).map(selectedValue => ( el.value === selectedValue)?.value} @@ -229,7 +230,7 @@ export function SelectComponent(props: SelectProps) { className={classes.chip} /> ))} -
+
) : ( {(value as any[]).length === 0 @@ -279,6 +280,6 @@ export function SelectComponent(props: SelectProps) { -
+ ); } diff --git a/packages/core-components/src/components/Select/static/ClosedDropdown.tsx b/packages/core-components/src/components/Select/static/ClosedDropdown.tsx index 7b0516d017..812afe34eb 100644 --- a/packages/core-components/src/components/Select/static/ClosedDropdown.tsx +++ b/packages/core-components/src/components/Select/static/ClosedDropdown.tsx @@ -21,11 +21,11 @@ import SvgIcon from '@material-ui/core/SvgIcon'; export type ClosedDropdownClassKey = 'icon'; const useStyles = makeStyles( - () => + theme => createStyles({ icon: { position: 'absolute', - right: '4px', + right: theme.spacing(0.5), pointerEvents: 'none', }, }), diff --git a/packages/core-components/src/components/Select/static/OpenedDropdown.tsx b/packages/core-components/src/components/Select/static/OpenedDropdown.tsx index 617288ded6..b87a00c26a 100644 --- a/packages/core-components/src/components/Select/static/OpenedDropdown.tsx +++ b/packages/core-components/src/components/Select/static/OpenedDropdown.tsx @@ -20,11 +20,11 @@ import SvgIcon from '@material-ui/core/SvgIcon'; export type OpenedDropdownClassKey = 'icon'; const useStyles = makeStyles( - () => + theme => createStyles({ icon: { position: 'absolute', - right: '4px', + right: theme.spacing(0.5), pointerEvents: 'none', }, }), diff --git a/packages/core-components/src/components/SimpleStepper/SimpleStepperFooter.tsx b/packages/core-components/src/components/SimpleStepper/SimpleStepperFooter.tsx index 0f7d604eb0..224b66533a 100644 --- a/packages/core-components/src/components/SimpleStepper/SimpleStepperFooter.tsx +++ b/packages/core-components/src/components/SimpleStepper/SimpleStepperFooter.tsx @@ -13,11 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useContext, ReactNode, PropsWithChildren } from 'react'; -import { makeStyles } from '@material-ui/core/styles'; +import Box from '@material-ui/core/Box'; import Button from '@material-ui/core/Button'; -import { StepActions } from './types'; +import { makeStyles } from '@material-ui/core/styles'; +import React, { PropsWithChildren, ReactNode, useContext } from 'react'; + import { VerticalStepperContext } from './SimpleStepper'; +import { StepActions } from './types'; export type SimpleStepperFooterClassKey = 'root'; @@ -145,7 +147,7 @@ export const SimpleStepperFooter = ({ }; return ( -
+ {[undefined, true].includes(actions.showBack) && stepIndex !== 0 && ( )} {children} -
+ ); }; diff --git a/packages/core-components/src/components/SimpleStepper/SimpleStepperStep.tsx b/packages/core-components/src/components/SimpleStepper/SimpleStepperStep.tsx index bb46163105..911ad2e73e 100644 --- a/packages/core-components/src/components/SimpleStepper/SimpleStepperStep.tsx +++ b/packages/core-components/src/components/SimpleStepper/SimpleStepperStep.tsx @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { PropsWithChildren } from 'react'; -import { makeStyles } from '@material-ui/core/styles'; +import Box from '@material-ui/core/Box'; import MuiStep from '@material-ui/core/Step'; import StepContent from '@material-ui/core/StepContent'; import StepLabel from '@material-ui/core/StepLabel'; +import { makeStyles } from '@material-ui/core/styles'; import Typography from '@material-ui/core/Typography'; +import React, { PropsWithChildren } from 'react'; + import { SimpleStepperFooter } from './SimpleStepperFooter'; import { StepProps } from './types'; @@ -40,11 +42,11 @@ export function SimpleStepperStep(props: PropsWithChildren) { // The end step is not a part of the stepper // It simply is the final screen with an option to have buttons such as reset or back return end ? ( -
+ {title} {children} -
+ ) : ( diff --git a/packages/core-components/src/components/Status/Status.tsx b/packages/core-components/src/components/Status/Status.tsx index a855fe3ebe..364193e383 100644 --- a/packages/core-components/src/components/Status/Status.tsx +++ b/packages/core-components/src/components/Status/Status.tsx @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { makeStyles } from '@material-ui/core/styles'; import { BackstageTheme } from '@backstage/theme'; +import { makeStyles } from '@material-ui/core/styles'; +import Typography from '@material-ui/core/Typography'; import classNames from 'classnames'; import React, { PropsWithChildren } from 'react'; @@ -31,12 +31,12 @@ export type StatusClassKey = const useStyles = makeStyles( theme => ({ status: { - fontWeight: 500, + fontWeight: theme.typography.fontWeightMedium, '&::before': { width: '0.7em', height: '0.7em', display: 'inline-block', - marginRight: 8, + marginRight: theme.spacing(1), borderRadius: '50%', content: '""', }, @@ -78,7 +78,8 @@ const useStyles = makeStyles( export function StatusOK(props: PropsWithChildren<{}>) { const classes = useStyles(props); return ( -