use HumanDuration in public incremental api

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2024-10-22 15:20:40 +02:00
parent bdd072c98f
commit 6cf91c5e74
6 changed files with 26 additions and 15 deletions
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor
---
Use `HumanDuration` for all duration needs in the public API, instead of `luxon` types. These are generally compatible, with a few caveats:
- If you scheduled things to run quarterly (`quarter` or `quarters`), you can use `{ months: 3 }` instead.
- If you used the singular nouns such as `year: 1`, use plurals instead (e.g. `years: 1`).
@@ -60,8 +60,8 @@
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/plugin-events-node": "workspace:^",
"@backstage/plugin-permission-common": "workspace:^",
"@backstage/types": "workspace:^",
"@types/express": "^4.17.6",
"@types/luxon": "^3.0.0",
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"knex": "^3.0.0",
@@ -71,6 +71,7 @@
"devDependencies": {
"@backstage/backend-defaults": "workspace:^",
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^"
"@backstage/cli": "workspace:^",
"@types/luxon": "^3.0.0"
}
}
@@ -9,10 +9,10 @@ import { BackendFeature } from '@backstage/backend-plugin-api';
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
import type { Config } from '@backstage/config';
import type { DeferredEntity } from '@backstage/plugin-catalog-node';
import type { DurationObjectUnits } from 'luxon';
import { EventParams } from '@backstage/plugin-events-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
import { HumanDuration } from '@backstage/types';
import { IncrementalEntityProvider as IncrementalEntityProvider_2 } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import { IncrementalEntityProviderOptions as IncrementalEntityProviderOptions_2 } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import type { Logger } from 'winston';
@@ -85,12 +85,12 @@ export interface IncrementalEntityProvider<TCursor, TContext> {
// @public (undocumented)
export interface IncrementalEntityProviderOptions {
backoff?: DurationObjectUnits[];
burstInterval: DurationObjectUnits;
burstLength: DurationObjectUnits;
backoff?: HumanDuration[];
burstInterval: HumanDuration;
burstLength: HumanDuration;
rejectEmptySourceCollections?: boolean;
rejectRemovalsAbovePercentage?: number;
restLength: DurationObjectUnits;
restLength: HumanDuration;
}
// @public
@@ -18,16 +18,17 @@ import type { DeferredEntity } from '@backstage/plugin-catalog-node';
import { IterationEngine, IterationEngineOptions } from '../types';
import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';
import { performance } from 'perf_hooks';
import { Duration, DurationObjectUnits } from 'luxon';
import { Duration } from 'luxon';
import { v4 } from 'uuid';
import { stringifyError } from '@backstage/errors';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import { HumanDuration } from '@backstage/types';
export class IncrementalIngestionEngine
implements IterationEngine, EventSubscriber
{
private readonly restLength: Duration;
private readonly backoff: DurationObjectUnits[];
private readonly backoff: HumanDuration[];
private manager: IncrementalIngestionDatabaseManager;
@@ -22,7 +22,6 @@ import type {
} from '@backstage/plugin-catalog-node';
import { EventParams } from '@backstage/plugin-events-node';
import type { PermissionEvaluator } from '@backstage/plugin-permission-common';
import type { DurationObjectUnits } from 'luxon';
import type { Logger } from 'winston';
import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager';
import {
@@ -31,6 +30,7 @@ import {
SchedulerService,
SchedulerServiceTaskFunction,
} from '@backstage/backend-plugin-api';
import { HumanDuration } from '@backstage/types';
/**
* Ingest entities into the catalog in bite-sized chunks.
@@ -145,27 +145,27 @@ export interface IncrementalEntityProviderOptions {
* Entities are ingested in bursts. This interval determines how
* much time to wait in between each burst.
*/
burstInterval: DurationObjectUnits;
burstInterval: HumanDuration;
/**
* Entities are ingested in bursts. This value determines how long
* to keep ingesting within each burst.
*/
burstLength: DurationObjectUnits;
burstLength: HumanDuration;
/**
* After a successful ingestion, the incremental entity provider
* will rest for this period of time before starting to ingest
* again.
*/
restLength: DurationObjectUnits;
restLength: HumanDuration;
/**
* In the event of an error during an ingestion burst, the backoff
* determines how soon it will be retried. E.g.
* `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]`
*/
backoff?: DurationObjectUnits[];
backoff?: HumanDuration[];
/**
* If an error occurs at a data source that results in a large
@@ -203,7 +203,7 @@ export interface IterationEngineOptions {
connection: EntityProviderConnection;
manager: IncrementalIngestionDatabaseManager;
provider: IncrementalEntityProvider<unknown, unknown>;
restLength: DurationObjectUnits;
restLength: HumanDuration;
ready: Promise<void>;
backoff?: IncrementalEntityProviderOptions['backoff'];
rejectRemovalsAbovePercentage?: number;
+1
View File
@@ -5859,6 +5859,7 @@ __metadata:
"@backstage/plugin-catalog-node": "workspace:^"
"@backstage/plugin-events-node": "workspace:^"
"@backstage/plugin-permission-common": "workspace:^"
"@backstage/types": "workspace:^"
"@types/express": ^4.17.6
"@types/luxon": ^3.0.0
express: ^4.17.1