From 2a8e3cc0b509427c1c6ac66cd45538f76ebc51b4 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Tue, 22 Nov 2022 12:45:02 -0500 Subject: [PATCH 1/2] refactor(stitcher): optimize to be more memory efficient Signed-off-by: Phil Kuang --- .changeset/eight-tables-hunt.md | 5 +++ .../catalog-backend/src/stitching/Stitcher.ts | 34 ++++++++----------- 2 files changed, 20 insertions(+), 19 deletions(-) create mode 100644 .changeset/eight-tables-hunt.md diff --git a/.changeset/eight-tables-hunt.md b/.changeset/eight-tables-hunt.md new file mode 100644 index 0000000000..bd1be0047f --- /dev/null +++ b/.changeset/eight-tables-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Optimize `Stitcher` process to be more memory efficient diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index cf65a14fee..3618d331ac 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -22,7 +22,6 @@ import { } from '@backstage/catalog-model'; import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; -import { uniqBy } from 'lodash'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { @@ -81,17 +80,12 @@ export class Stitcher { // one row (except in abnormal cases where the stitch was invoked for // something that didn't exist at all, in which case it's zero rows). // The join with the temporary incoming_references still gives one row. - // The only result set "expanding" join is the one with relations, so - // the output should be at least one row (if zero or one relations were - // found), or at most the same number of rows as relations. - const result: Array<{ + const processedResult: Array<{ entityId: string; processedEntity?: string; errors: string; incomingReferenceCount: string | number; previousHash?: string; - relationType?: string; - relationTarget?: string; }> = await this.database .with('incoming_references', function incomingReferences(builder) { return builder @@ -105,18 +99,24 @@ export class Stitcher { errors: 'refresh_state.errors', incomingReferenceCount: 'incoming_references.count', previousHash: 'final_entities.hash', - relationType: 'relations.type', - relationTarget: 'relations.target_entity_ref', }) .from('refresh_state') .where({ 'refresh_state.entity_ref': entityRef }) .crossJoin(this.database.raw('incoming_references')) .leftOuterJoin('final_entities', { 'final_entities.entity_id': 'refresh_state.entity_id', + }); + + const relationsResult: Array<{ + relationType: string; + relationTarget: string; + }> = await this.database + .distinct({ + relationType: 'type', + relationTarget: 'target_entity_ref', }) - .leftOuterJoin('relations', { - 'relations.source_entity_ref': 'refresh_state.entity_ref', - }) + .from('relations') + .where({ source_entity_ref: entityRef }) .orderBy('relationType', 'asc') .orderBy('relationTarget', 'asc'); @@ -124,7 +124,7 @@ export class Stitcher { // matching row even in the refresh_state. This can happen for example // if we emit a relation to something that hasn't been ingested yet. // It's safe to ignore this stitch attempt in that case. - if (!result.length) { + if (!processedResult.length) { this.logger.error( `Unable to stitch ${entityRef}, item does not exist in refresh state table`, ); @@ -137,7 +137,7 @@ export class Stitcher { errors, incomingReferenceCount, previousHash, - } = result[0]; + } = processedResult[0]; // If there was no processed entity in place, the target hasn't been // through the processing steps yet. It's safe to ignore this stitch @@ -177,11 +177,7 @@ export class Stitcher { // TODO: entityRef is lower case and should be uppercase in the final // result - const uniqueRelationRows = uniqBy( - result, - r => `${r.relationType}:${r.relationTarget}`, - ); - entity.relations = uniqueRelationRows + entity.relations = relationsResult .filter(row => row.relationType /* exclude null row, if relevant */) .map(row => ({ type: row.relationType!, From dba451bf2e32bb8db02f11f396b7aaf89839fc27 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Sun, 27 Nov 2022 14:01:24 -0500 Subject: [PATCH 2/2] refactor(stitcher): parallelize DB queries Signed-off-by: Phil Kuang --- .../catalog-backend/src/stitching/Stitcher.ts | 70 ++++++++----------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 3618d331ac..0da8e39694 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -80,45 +80,37 @@ export class Stitcher { // one row (except in abnormal cases where the stitch was invoked for // something that didn't exist at all, in which case it's zero rows). // The join with the temporary incoming_references still gives one row. - const processedResult: Array<{ - entityId: string; - processedEntity?: string; - errors: string; - incomingReferenceCount: string | number; - previousHash?: string; - }> = await this.database - .with('incoming_references', function incomingReferences(builder) { - return builder - .from('refresh_state_references') - .where({ target_entity_ref: entityRef }) - .count({ count: '*' }); - }) - .select({ - entityId: 'refresh_state.entity_id', - processedEntity: 'refresh_state.processed_entity', - errors: 'refresh_state.errors', - incomingReferenceCount: 'incoming_references.count', - previousHash: 'final_entities.hash', - }) - .from('refresh_state') - .where({ 'refresh_state.entity_ref': entityRef }) - .crossJoin(this.database.raw('incoming_references')) - .leftOuterJoin('final_entities', { - 'final_entities.entity_id': 'refresh_state.entity_id', - }); - - const relationsResult: Array<{ - relationType: string; - relationTarget: string; - }> = await this.database - .distinct({ - relationType: 'type', - relationTarget: 'target_entity_ref', - }) - .from('relations') - .where({ source_entity_ref: entityRef }) - .orderBy('relationType', 'asc') - .orderBy('relationTarget', 'asc'); + const [processedResult, relationsResult] = await Promise.all([ + this.database + .with('incoming_references', function incomingReferences(builder) { + return builder + .from('refresh_state_references') + .where({ target_entity_ref: entityRef }) + .count({ count: '*' }); + }) + .select({ + entityId: 'refresh_state.entity_id', + processedEntity: 'refresh_state.processed_entity', + errors: 'refresh_state.errors', + incomingReferenceCount: 'incoming_references.count', + previousHash: 'final_entities.hash', + }) + .from('refresh_state') + .where({ 'refresh_state.entity_ref': entityRef }) + .crossJoin(this.database.raw('incoming_references')) + .leftOuterJoin('final_entities', { + 'final_entities.entity_id': 'refresh_state.entity_id', + }), + this.database + .distinct({ + relationType: 'type', + relationTarget: 'target_entity_ref', + }) + .from('relations') + .where({ source_entity_ref: entityRef }) + .orderBy('relationType', 'asc') + .orderBy('relationTarget', 'asc'), + ]); // If there were no rows returned, it would mean that there was no // matching row even in the refresh_state. This can happen for example