From 5f6c806378ba9899ab218a0dd6e9e253043e26ff Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 11 Jun 2021 11:01:51 +0200 Subject: [PATCH 1/4] catalog/next: Make refresh interval configurable Signed-off-by: Johan Haals --- .../src/next/NextCatalogBuilder.ts | 17 ++++++++++++++++- .../next/database/DefaultProcessingDatabase.ts | 11 +++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 7a74576ea7..6685296312 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -108,6 +108,7 @@ export class NextCatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; + private refreshIntervalSeconds = 100; constructor(env: CatalogEnvironment) { this.env = env; @@ -136,6 +137,16 @@ export class NextCatalogBuilder { return this; } + /** + * Refresh interval determines how often entities should be refreshed. + * The default refresh duration is 100, setting this too low will potentially + * deplete request quotas to upstream services. + */ + setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder { + this.refreshIntervalSeconds = seconds; + return this; + } + /** * Sets what policies to use for validation of entities between the pre- * processing and post-processing stages. All such policies must pass for the @@ -252,7 +263,11 @@ export class NextCatalogBuilder { const db = new CommonDatabase(dbClient, logger); - const processingDatabase = new DefaultProcessingDatabase(dbClient, logger); + const processingDatabase = new DefaultProcessingDatabase( + dbClient, + logger, + this.refreshIntervalSeconds, + ); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ processors, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 3569d4dcc6..10c213d218 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -46,6 +46,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { constructor( private readonly database: Knex, private readonly logger: Logger, + private readonly refreshIntervalSeconds: number, ) {} async updateProcessedEntity( @@ -377,8 +378,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .update({ next_update_at: tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [`100 seconds`]) - : tx.raw(`now() + interval '100 seconds'`), + ? tx.raw(`datetime('now', ?)`, [ + `${this.refreshIntervalSeconds} seconds`, + ]) + : tx.raw( + `now() + interval '${Number( + this.refreshIntervalSeconds, + )} seconds'`, + ), }); return { From 5b721e1ebc030bb47ec22c24426d4e82f647aaa8 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 11 Jun 2021 11:23:08 +0200 Subject: [PATCH 2/4] chore: pass constructor arguments Signed-off-by: Johan Haals --- .../src/next/database/DefaultProcessingDatabase.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 7d98a32377..5843ef6c34 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -39,7 +39,7 @@ describe('Default Processing Database', () => { await DatabaseManager.createDatabase(knex); return { knex, - db: new DefaultProcessingDatabase(knex, logger), + db: new DefaultProcessingDatabase(knex, logger, 100), }; } From 9600e5e6696439235ea67af3a344e6185e7127ae Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 11 Jun 2021 11:52:37 +0200 Subject: [PATCH 3/4] chore: Update constructor args to object Signed-off-by: Johan Haals --- .../src/next/NextCatalogBuilder.ts | 8 ++++---- .../database/DefaultProcessingDatabase.test.ts | 6 +++++- .../next/database/DefaultProcessingDatabase.ts | 18 ++++++++++-------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 6685296312..944036e48c 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -263,11 +263,11 @@ export class NextCatalogBuilder { const db = new CommonDatabase(dbClient, logger); - const processingDatabase = new DefaultProcessingDatabase( - dbClient, + const processingDatabase = new DefaultProcessingDatabase({ + database: dbClient, logger, - this.refreshIntervalSeconds, - ); + refreshIntervalSeconds: this.refreshIntervalSeconds, + }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ processors, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 5843ef6c34..0f83311944 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -39,7 +39,11 @@ describe('Default Processing Database', () => { await DatabaseManager.createDatabase(knex); return { knex, - db: new DefaultProcessingDatabase(knex, logger, 100), + db: new DefaultProcessingDatabase({ + database: knex, + logger, + refreshIntervalSeconds: 100, + }), }; } diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 10c213d218..f4804f9118 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -44,9 +44,11 @@ const BATCH_SIZE = 50; export class DefaultProcessingDatabase implements ProcessingDatabase { constructor( - private readonly database: Knex, - private readonly logger: Logger, - private readonly refreshIntervalSeconds: number, + private readonly options: { + database: Knex; + logger: Logger; + refreshIntervalSeconds: number; + }, ) {} async updateProcessedEntity( @@ -273,7 +275,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .whereIn('target_entity_ref', toRemove) .delete(); - this.logger.debug( + this.options.logger.debug( `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, ); } @@ -379,11 +381,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { next_update_at: tx.client.config.client === 'sqlite3' ? tx.raw(`datetime('now', ?)`, [ - `${this.refreshIntervalSeconds} seconds`, + `${this.options.refreshIntervalSeconds} seconds`, ]) : tx.raw( `now() + interval '${Number( - this.refreshIntervalSeconds, + this.options.refreshIntervalSeconds, )} seconds'`, ), }); @@ -413,7 +415,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { try { let result: T | undefined = undefined; - await this.database.transaction( + await this.options.database.transaction( async tx => { // We can't return here, as knex swallows the return type in case the transaction is rolled back: // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 @@ -427,7 +429,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { return result!; } catch (e) { - this.logger.debug(`Error during transaction, ${e}`); + this.options.logger.debug(`Error during transaction, ${e}`); if ( /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) || From db17fd734e4caaf70f5a9b335aea3ba4b5d9729c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 11 Jun 2021 11:52:59 +0200 Subject: [PATCH 4/4] Add changeset Signed-off-by: Johan Haals --- .changeset/kind-tools-kneel.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/kind-tools-kneel.md diff --git a/.changeset/kind-tools-kneel.md b/.changeset/kind-tools-kneel.md new file mode 100644 index 0000000000..6296ac7011 --- /dev/null +++ b/.changeset/kind-tools-kneel.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Make refresh interval configurable for the `NextCatalogBuilder` using `.setRefreshIntervalSeconds()`. + +Change `DefaultProcessingDatabase` constructor to accept an options object instead of individual arguments.