From 22f775d19faf816461bf591e9a75d21db55f6b71 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 13 Aug 2021 14:40:33 +0200 Subject: [PATCH 1/9] Catalog: Add refresh spread functionality Signed-off-by: Johan Haals --- plugins/catalog-backend/package.json | 3 +- .../src/next/NextCatalogBuilder.ts | 11 +++++ .../DefaultProcessingDatabase.test.ts | 44 +++++++++++++++++++ .../database/DefaultProcessingDatabase.ts | 26 ++++++----- 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 0bf5a08dfd..f095f39785 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -73,7 +73,8 @@ "msw": "^0.29.0", "sqlite3": "^5.0.1", "supertest": "^6.1.3", - "wait-for-expect": "^3.0.2" + "wait-for-expect": "^3.0.2", + "luxon": "^2.0.2" }, "files": [ "dist", diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..fe7df180ee 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -113,6 +113,7 @@ export class NextCatalogBuilder { private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; private refreshIntervalSeconds = 100; + private refreshSpreadSeconds = { min: 10, max: 60 }; constructor(env: CatalogEnvironment) { this.env = env; @@ -152,6 +153,15 @@ export class NextCatalogBuilder { return this; } + /** + * Refresh spread configures configures the minimum and maximum number of seconds + * to wait between refreshes in addition to the configured refresh interval. + */ + setRefreshSpreadSeconds(min: number, max: number): NextCatalogBuilder { + this.refreshSpreadSeconds = { min, max }; + return this; + } + /** * Sets what policies to use for validation of entities between the pre- * processing and post-processing stages. All such policies must pass for the @@ -286,6 +296,7 @@ export class NextCatalogBuilder { database: dbClient, logger, refreshIntervalSeconds: this.refreshIntervalSeconds, + refreshSpreadSeconds: this.refreshSpreadSeconds, }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 5103c25dbc..690dcc1b7b 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -20,7 +20,11 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Knex } from 'knex'; import * as uuid from 'uuid'; +<<<<<<< HEAD import { Logger } from 'winston'; +======= +import { DateTime } from 'luxon'; +>>>>>>> d31a82869 (Catalog: Add refresh spread functionality) import { DatabaseManager } from './DatabaseManager'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { @@ -47,6 +51,7 @@ describe('Default Processing Database', () => { database: knex, logger, refreshIntervalSeconds: 100, + refreshSpreadSeconds: { min: 10, max: 60 }, }), }; } @@ -960,5 +965,44 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'should update the next_refresh interval with a timestamp that includes refresh spread, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + const entity = JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + } as Entity); + await knex('refresh_state').insert({ + entity_id: '2', + entity_ref: 'location:default/new-root', + unprocessed_entity: entity, + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + await db.transaction(async tx => { + // Result does not include the updated timestamp + await db.getProcessableEntities(tx, { + processBatchSize: 1, + }); + }); + const now = DateTime.local(); + const result = await knex('refresh_state') + .where('entity_ref', 'location:default/new-root') + .select(); + const nextUpdate = DateTime.fromSQL(result[0].next_update_at, { + zone: 'utc', + }); + expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( + 110, + ); + }, + 60_000, + ); }); }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 177bf979ab..926423c848 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -49,6 +49,10 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { database: Knex; logger: Logger; refreshIntervalSeconds: number; + refreshSpreadSeconds: { + min: number; + max: number; + }; }, ) {} @@ -447,6 +451,17 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } + // Returns the next update timestamp by combining refresh interval and refresh spread + private getNextUpdateAt(tx: Knex.Transaction): Knex.MaybeRawColumn { + const { min, max } = this.options.refreshSpreadSeconds; + const refreshSpread = Math.floor(Math.random() * (max - min + 1)) + min; + const nextUpdate = this.options.refreshIntervalSeconds + refreshSpread; + + return tx.client.config.client === 'sqlite3' + ? tx.raw(`datetime('now', ?)`, [`${nextUpdate} seconds`]) + : tx.raw(`now() + interval '${Number(nextUpdate)} seconds'`); + } + async getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, @@ -473,16 +488,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { items.map(i => i.entity_ref), ) .update({ - next_update_at: - tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [ - `${this.options.refreshIntervalSeconds} seconds`, - ]) - : tx.raw( - `now() + interval '${Number( - this.options.refreshIntervalSeconds, - )} seconds'`, - ), + next_update_at: this.getNextUpdateAt(tx), }); return { From fe960ad0fd60929a2997771d73c049ef404a0e0b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 13 Aug 2021 15:31:07 +0200 Subject: [PATCH 2/9] catalog-backend: Add refresh interval function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- .changeset/early-ducks-stare.md | 8 +++++ .../src/next/NextCatalogBuilder.ts | 30 +++++++++++----- .../DefaultProcessingDatabase.test.ts | 9 +++-- .../database/DefaultProcessingDatabase.ts | 27 ++++++-------- plugins/catalog-backend/src/next/index.ts | 2 ++ plugins/catalog-backend/src/next/refresh.ts | 36 +++++++++++++++++++ 6 files changed, 83 insertions(+), 29 deletions(-) create mode 100644 .changeset/early-ducks-stare.md create mode 100644 plugins/catalog-backend/src/next/refresh.ts diff --git a/.changeset/early-ducks-stare.md b/.changeset/early-ducks-stare.md new file mode 100644 index 0000000000..58b1525e25 --- /dev/null +++ b/.changeset/early-ducks-stare.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should +help to smooth out bursts that occur when a lot of entities are scheduled for refresh at the same second. + +Custom `RefreshIntervalFunction` can be implemented and passed to the CatalogBuilder using `.setInterval(fn)` diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fe7df180ee..b277f0b16a 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -75,6 +75,10 @@ import { DefaultLocationStore } from './DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from './stitching/Stitcher'; +import { + createRandomRefreshInterval, + RefreshIntervalFunction, +} from './refresh'; export type CatalogEnvironment = { logger: Logger; @@ -112,8 +116,11 @@ export class NextCatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshIntervalSeconds = 100; - private refreshSpreadSeconds = { min: 10, max: 60 }; + private refreshInterval: RefreshIntervalFunction = + createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }); constructor(env: CatalogEnvironment) { this.env = env; @@ -145,11 +152,15 @@ export class NextCatalogBuilder { /** * Refresh interval determines how often entities should be refreshed. - * The default refresh duration is 100, setting this too low will potentially - * deplete request quotas to upstream services. + * Seconds provided will be multiplied by 1.5 + * The default refresh duration is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. */ setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder { - this.refreshIntervalSeconds = seconds; + this.refreshInterval = createRandomRefreshInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); return this; } @@ -157,8 +168,10 @@ export class NextCatalogBuilder { * Refresh spread configures configures the minimum and maximum number of seconds * to wait between refreshes in addition to the configured refresh interval. */ - setRefreshSpreadSeconds(min: number, max: number): NextCatalogBuilder { - this.refreshSpreadSeconds = { min, max }; + setRefreshInterval( + refreshInterval: RefreshIntervalFunction, + ): NextCatalogBuilder { + this.refreshInterval = refreshInterval; return this; } @@ -295,8 +308,7 @@ export class NextCatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshIntervalSeconds: this.refreshIntervalSeconds, - refreshSpreadSeconds: this.refreshSpreadSeconds, + refreshInterval: this.refreshInterval, }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 690dcc1b7b..5b717ee36e 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -32,6 +32,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; +import { createRandomRefreshInterval } from '../refresh'; describe('Default Processing Database', () => { const defaultLogger = getVoidLogger(); @@ -50,8 +51,10 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshIntervalSeconds: 100, - refreshSpreadSeconds: { min: 10, max: 60 }, + refreshInterval: createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }), }), }; } @@ -999,7 +1002,7 @@ describe('Default Processing Database', () => { zone: 'utc', }); expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( - 110, + 100, ); }, 60_000, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 926423c848..203ca3efbf 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -23,6 +23,7 @@ import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; import { Transaction } from '../../database'; import { DeferredEntity } from '../processing/types'; +import { RefreshIntervalFunction } from '../refresh'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, @@ -48,11 +49,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { private readonly options: { database: Knex; logger: Logger; - refreshIntervalSeconds: number; - refreshSpreadSeconds: { - min: number; - max: number; - }; + refreshInterval: RefreshIntervalFunction; }, ) {} @@ -451,17 +448,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } - // Returns the next update timestamp by combining refresh interval and refresh spread - private getNextUpdateAt(tx: Knex.Transaction): Knex.MaybeRawColumn { - const { min, max } = this.options.refreshSpreadSeconds; - const refreshSpread = Math.floor(Math.random() * (max - min + 1)) + min; - const nextUpdate = this.options.refreshIntervalSeconds + refreshSpread; - - return tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [`${nextUpdate} seconds`]) - : tx.raw(`now() + interval '${Number(nextUpdate)} seconds'`); - } - async getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, @@ -488,7 +474,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { items.map(i => i.entity_ref), ) .update({ - next_update_at: this.getNextUpdateAt(tx), + next_update_at: + tx.client.config.client === 'sqlite3' + ? tx.raw(`datetime('now', ?)`, [ + `${this.options.refreshInterval()} seconds`, + ]) + : tx.raw( + `now() + interval '${this.options.refreshInterval()} seconds'`, + ), }); return { diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index 8c1921ae8b..a27174c415 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -18,3 +18,5 @@ export { NextCatalogBuilder } from './NextCatalogBuilder'; export { createNextRouter } from './NextRouter'; export * from './processing'; export * from './stitching'; +export type { RefreshIntervalFunction } from './refresh'; +export { createRandomRefreshInterval } from './refresh'; diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts new file mode 100644 index 0000000000..d9f56c3254 --- /dev/null +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Function that returns the catalog refresh interval in seconds. + */ +export type RefreshIntervalFunction = () => number; + +/** + * @param {number} options.minSeconds The minimum number of seconds between refreshes + * @param {number} options.maxSeconds The maximum number of seconds between refreshes + * @returns {RefreshIntervalFunction} that provides the next refresh interval + * + */ +export function createRandomRefreshInterval(options: { + minSeconds: number; + maxSeconds: number; +}): RefreshIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +} From 8fbf7b74522b880971574e90b1b7eaf0e95e13c9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 16 Aug 2021 09:43:53 +0200 Subject: [PATCH 3/9] API reports Signed-off-by: Johan Haals --- packages/catalog-client/api-report.md | 53 +++++++++++++++------------ plugins/catalog-backend/api-report.md | 30 ++++++++++++++- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 173af9aef3..2e39dfd07f 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -25,11 +25,6 @@ export type AddLocationResponse = { entities: Entity[]; }; -// Warning: (ae-missing-release-tag) "CATALOG_FILTER_EXISTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export const CATALOG_FILTER_EXISTS: unique symbol; - // Warning: (ae-missing-release-tag) "CatalogApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -76,50 +71,59 @@ export interface CatalogApi { ): Promise; } +// Warning: (ae-forgotten-export) The symbol "CatalogApi" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "CatalogClient" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class CatalogClient implements CatalogApi { - constructor(options: { discoveryApi: DiscoveryApi }); +export class CatalogClient implements CatalogApi_2 { + constructor(options: { discoveryApi: DiscoveryApi_2 }); + // Warning: (ae-forgotten-export) The symbol "AddLocationRequest" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "AddLocationResponse" needs to be exported by the entry point index.d.ts + // // (undocumented) addLocation( - { type, target, dryRun, presence }: AddLocationRequest, - options?: CatalogRequestOptions, - ): Promise; + { type, target, dryRun, presence }: AddLocationRequest_2, + options?: CatalogRequestOptions_2, + ): Promise; + // Warning: (ae-forgotten-export) The symbol "CatalogEntitiesRequest" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "CatalogListResponse" needs to be exported by the entry point index.d.ts + // // (undocumented) getEntities( - request?: CatalogEntitiesRequest, - options?: CatalogRequestOptions, - ): Promise>; + request?: CatalogEntitiesRequest_2, + options?: CatalogRequestOptions_2, + ): Promise>; // (undocumented) getEntityByName( compoundName: EntityName, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; // (undocumented) getLocationByEntity( entity: Entity, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; + // Warning: (ae-forgotten-export) The symbol "CatalogRequestOptions" needs to be exported by the entry point index.d.ts + // // (undocumented) getLocationById( id: string, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; // (undocumented) getOriginLocationByEntity( entity: Entity, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; // (undocumented) removeEntityByUid( uid: string, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; // (undocumented) removeLocationById( id: string, - options?: CatalogRequestOptions, + options?: CatalogRequestOptions_2, ): Promise; } @@ -128,8 +132,8 @@ export class CatalogClient implements CatalogApi { // @public (undocumented) export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; @@ -148,11 +152,12 @@ export type CatalogRequestOptions = { token?: string; }; -// Warning: (ae-missing-release-tag) "ENTITY_STATUS_CATALOG_PROCESSING_TYPE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "DiscoveryApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = - 'backstage.io/catalog-processing'; +export type DiscoveryApi = { + getBaseUrl(pluginId: string): Promise; +}; // Warnings were encountered during analysis: // diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7286855d4e..8f0d6c32e9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -419,6 +419,24 @@ export function createNextRouter( options: RouterOptions_2, ): Promise; +// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag +// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" +// Warning: (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' +// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag +// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" +// Warning: (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' +// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag +// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" +// Warning: (ae-missing-release-tag) "createRandomRefreshInterval" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function createRandomRefreshInterval(options: { + minSeconds: number; + maxSeconds: number; +}): RefreshIntervalFunction; + // Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -914,6 +932,9 @@ export class NextCatalogBuilder { key: string, resolver: PlaceholderResolver, ): NextCatalogBuilder; + setRefreshInterval( + refreshInterval: RefreshIntervalFunction, + ): NextCatalogBuilder; setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder; } @@ -987,6 +1008,11 @@ export type RecursivePartial = { : T[P]; }; +// Warning: (ae-missing-release-tag) "RefreshIntervalFunction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type RefreshIntervalFunction = () => number; + // Warning: (ae-missing-release-tag) "relation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -1107,8 +1133,8 @@ export class UrlReaderProcessor implements CatalogProcessor { // src/ingestion/types.d.ts:41:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/ingestion/types.d.ts:49:5 - (ae-forgotten-export) The symbol "AnalyzeLocationExistingEntity" needs to be exported by the entry point index.d.ts // src/ingestion/types.d.ts:50:5 - (ae-forgotten-export) The symbol "AnalyzeLocationGenerateEntity" needs to be exported by the entry point index.d.ts -// src/next/NextCatalogBuilder.d.ts:140:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts -// src/next/NextCatalogBuilder.d.ts:141:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts +// src/next/NextCatalogBuilder.d.ts:147:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts +// src/next/NextCatalogBuilder.d.ts:148:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts // src/next/processing/types.d.ts:11:5 - (ae-forgotten-export) The symbol "DeferredEntity" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) From 2fc711c675dac56217c96239c6ac7d07d38a1d40 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 16 Aug 2021 09:58:57 +0200 Subject: [PATCH 4/9] cleanup bad imports Signed-off-by: Johan Haals --- .../src/next/database/DefaultProcessingDatabase.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 5b717ee36e..09647b980c 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -20,11 +20,8 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Knex } from 'knex'; import * as uuid from 'uuid'; -<<<<<<< HEAD import { Logger } from 'winston'; -======= import { DateTime } from 'luxon'; ->>>>>>> d31a82869 (Catalog: Add refresh spread functionality) import { DatabaseManager } from './DatabaseManager'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { From 928c9ba27bf24fa852821e166d0c9434b08cb308 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 17 Aug 2021 09:28:36 +0200 Subject: [PATCH 5/9] API reports Signed-off-by: Johan Haals --- packages/catalog-client/api-report.md | 53 ++++++++++++--------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 2e39dfd07f..173af9aef3 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -25,6 +25,11 @@ export type AddLocationResponse = { entities: Entity[]; }; +// Warning: (ae-missing-release-tag) "CATALOG_FILTER_EXISTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const CATALOG_FILTER_EXISTS: unique symbol; + // Warning: (ae-missing-release-tag) "CatalogApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -71,59 +76,50 @@ export interface CatalogApi { ): Promise; } -// Warning: (ae-forgotten-export) The symbol "CatalogApi" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "CatalogClient" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class CatalogClient implements CatalogApi_2 { - constructor(options: { discoveryApi: DiscoveryApi_2 }); - // Warning: (ae-forgotten-export) The symbol "AddLocationRequest" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "AddLocationResponse" needs to be exported by the entry point index.d.ts - // +export class CatalogClient implements CatalogApi { + constructor(options: { discoveryApi: DiscoveryApi }); // (undocumented) addLocation( - { type, target, dryRun, presence }: AddLocationRequest_2, - options?: CatalogRequestOptions_2, - ): Promise; - // Warning: (ae-forgotten-export) The symbol "CatalogEntitiesRequest" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "CatalogListResponse" needs to be exported by the entry point index.d.ts - // + { type, target, dryRun, presence }: AddLocationRequest, + options?: CatalogRequestOptions, + ): Promise; // (undocumented) getEntities( - request?: CatalogEntitiesRequest_2, - options?: CatalogRequestOptions_2, - ): Promise>; + request?: CatalogEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise>; // (undocumented) getEntityByName( compoundName: EntityName, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; // (undocumented) getLocationByEntity( entity: Entity, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; - // Warning: (ae-forgotten-export) The symbol "CatalogRequestOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) getLocationById( id: string, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; // (undocumented) getOriginLocationByEntity( entity: Entity, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; // (undocumented) removeEntityByUid( uid: string, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; // (undocumented) removeLocationById( id: string, - options?: CatalogRequestOptions_2, + options?: CatalogRequestOptions, ): Promise; } @@ -132,8 +128,8 @@ export class CatalogClient implements CatalogApi_2 { // @public (undocumented) export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; @@ -152,12 +148,11 @@ export type CatalogRequestOptions = { token?: string; }; -// Warning: (ae-missing-release-tag) "DiscoveryApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "ENTITY_STATUS_CATALOG_PROCESSING_TYPE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export type DiscoveryApi = { - getBaseUrl(pluginId: string): Promise; -}; +export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = + 'backstage.io/catalog-processing'; // Warnings were encountered during analysis: // From 4f685aac962f2ad2db65adb15125c2093775056d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 17 Aug 2021 13:17:55 +0200 Subject: [PATCH 6/9] Update comments Signed-off-by: Johan Haals --- plugins/catalog-backend/api-report.md | 10 +--------- plugins/catalog-backend/src/next/NextCatalogBuilder.ts | 4 ++-- plugins/catalog-backend/src/next/refresh.ts | 4 +--- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 8f0d6c32e9..a6e67215c9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -419,19 +419,11 @@ export function createNextRouter( options: RouterOptions_2, ): Promise; -// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag -// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" -// Warning: (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' -// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag -// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" -// Warning: (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" // Warning: (ae-missing-release-tag) "createRandomRefreshInterval" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public (undocumented) +// @public export function createRandomRefreshInterval(options: { minSeconds: number; maxSeconds: number; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index b277f0b16a..675e170727 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -165,8 +165,8 @@ export class NextCatalogBuilder { } /** - * Refresh spread configures configures the minimum and maximum number of seconds - * to wait between refreshes in addition to the configured refresh interval. + * Overwrites the default refresh interval function used to spread + * entity updates in the catalog. */ setRefreshInterval( refreshInterval: RefreshIntervalFunction, diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts index d9f56c3254..aae992f391 100644 --- a/plugins/catalog-backend/src/next/refresh.ts +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -20,10 +20,8 @@ export type RefreshIntervalFunction = () => number; /** - * @param {number} options.minSeconds The minimum number of seconds between refreshes - * @param {number} options.maxSeconds The maximum number of seconds between refreshes + * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. * @returns {RefreshIntervalFunction} that provides the next refresh interval - * */ export function createRandomRefreshInterval(options: { minSeconds: number; From d155fedfc4e9ac8e38d792b3161c714fbc282f21 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 17 Aug 2021 13:46:37 +0200 Subject: [PATCH 7/9] Floor random number Signed-off-by: Johan Haals --- plugins/catalog-backend/src/next/refresh.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts index aae992f391..caec7e9548 100644 --- a/plugins/catalog-backend/src/next/refresh.ts +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -29,6 +29,6 @@ export function createRandomRefreshInterval(options: { }): RefreshIntervalFunction { const { minSeconds, maxSeconds } = options; return () => { - return Math.random() * (maxSeconds - minSeconds) + minSeconds; + return Math.floor(Math.random() * (maxSeconds - minSeconds) + minSeconds); }; } From ba01a6bcdfc918d53c2494482bb44a70f6cb4aa2 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 17 Aug 2021 14:59:07 +0200 Subject: [PATCH 8/9] Parse date correctly, drop math.Floor Signed-off-by: Johan Haals --- .../DefaultProcessingDatabase.test.ts | 19 ++++++++++++++++--- plugins/catalog-backend/src/next/refresh.ts | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 09647b980c..df9c08ad6c 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -66,6 +66,21 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; + const parseDate = (date: string | Date): DateTime => { + const parsedDate = + typeof date === 'string' + ? DateTime.fromSQL(date, { zone: 'UTC' }) + : DateTime.fromJSDate(date); + + if (!parsedDate.isValid) { + throw new Error( + `Failed to parse date, reason: ${parsedDate.invalidReason}, explanation: ${parsedDate.invalidExplanation}`, + ); + } + + return parsedDate; + }; + describe('addUprocessedEntities', () => { function mockEntity(name: string, type: string): Entity { return { @@ -995,9 +1010,7 @@ describe('Default Processing Database', () => { const result = await knex('refresh_state') .where('entity_ref', 'location:default/new-root') .select(); - const nextUpdate = DateTime.fromSQL(result[0].next_update_at, { - zone: 'utc', - }); + const nextUpdate = parseDate(result[0].next_update_at); expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( 100, ); diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts index caec7e9548..aae992f391 100644 --- a/plugins/catalog-backend/src/next/refresh.ts +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -29,6 +29,6 @@ export function createRandomRefreshInterval(options: { }): RefreshIntervalFunction { const { minSeconds, maxSeconds } = options; return () => { - return Math.floor(Math.random() * (maxSeconds - minSeconds) + minSeconds); + return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } From db83840a65cb93a9c6061ef83dfcd4dcdb911798 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 18 Aug 2021 11:06:01 +0200 Subject: [PATCH 9/9] Update .changeset/early-ducks-stare.md Signed-off-by: Johan Haals Co-authored-by: Patrik Oldsberg --- .changeset/early-ducks-stare.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/early-ducks-stare.md b/.changeset/early-ducks-stare.md index 58b1525e25..1fd44f2755 100644 --- a/.changeset/early-ducks-stare.md +++ b/.changeset/early-ducks-stare.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should