diff --git a/.changeset/loud-rockets-reply.md b/.changeset/loud-rockets-reply.md index 803521c97b..85ed77f51f 100644 --- a/.changeset/loud-rockets-reply.md +++ b/.changeset/loud-rockets-reply.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-incremental-ingestion-backend': minor +'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor --- Introduces incremental entity providers, which are used for streaming very large data sources into the catalog. diff --git a/plugins/incremental-ingestion-backend/.eslintrc.js b/plugins/catalog-backend-module-incremental-ingestion/.eslintrc.js similarity index 100% rename from plugins/incremental-ingestion-backend/.eslintrc.js rename to plugins/catalog-backend-module-incremental-ingestion/.eslintrc.js diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md similarity index 93% rename from plugins/incremental-ingestion-backend/README.md rename to plugins/catalog-backend-module-incremental-ingestion/README.md index 5df35684d5..d14e33ea53 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -1,6 +1,6 @@ -# `@backstage/plugin-incremental-ingestion-backend` +# `@backstage/plugin-catalog-backend-module-incremental-ingestion` -The Incremental Ingestion Backend plugin provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. +The Incremental Ingestion catalog backend module provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. ## Why did we create it? @@ -42,8 +42,8 @@ The Incremental Entity Provider backend is designed for data sources that provid ## Installation -1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn workspace backend add @backstage/plugin-incremental-ingestion-backend` -2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. +1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn workspace backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` +2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. ```ts const builder = CatalogBuilder.create(env); @@ -70,7 +70,7 @@ The Incremental Entity Provider backend is designed for data sources that provid ```ts import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; - import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; + import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { Router } from 'express'; import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; @@ -164,7 +164,7 @@ These are the only 3 methods that you need to implement. `getProviderName()` is import { IncrementalEntityProvider, EntityIteratorResult, -} from '@backstage/plugin-incremental-ingestion-backend'; +} from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; // this will include your pagination information, let's say our API accepts a `page` parameter. // In this case, the cursor will include `page` diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md similarity index 85% rename from plugins/incremental-ingestion-backend/api-report.md rename to plugins/catalog-backend-module-incremental-ingestion/api-report.md index ff635871aa..49eb143a4d 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -1,10 +1,11 @@ -## API Report File for "@backstage/plugin-incremental-ingestion-backend" +## API Report File for "@backstage/plugin-catalog-backend-module-incremental-ingestion" > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts /// +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; @@ -68,6 +69,14 @@ export interface IncrementalEntityProviderOptions { restLength: DurationObjectUnits; } +// @alpha +export const incrementalIngestionEntityProviderCatalogModule: (options: { + providers: { + provider: IncrementalEntityProvider; + options: IncrementalEntityProviderOptions; + }[]; +}) => BackendFeature; + // @public (undocumented) export type PluginEnvironment = { logger: Logger; diff --git a/plugins/incremental-ingestion-backend/knexfile.js b/plugins/catalog-backend-module-incremental-ingestion/knexfile.js similarity index 100% rename from plugins/incremental-ingestion-backend/knexfile.js rename to plugins/catalog-backend-module-incremental-ingestion/knexfile.js diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/catalog-backend-module-incremental-ingestion/migrations/20221116073152_init.js similarity index 100% rename from plugins/incremental-ingestion-backend/migrations/20221116073152_init.js rename to plugins/catalog-backend-module-incremental-ingestion/migrations/20221116073152_init.js diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json similarity index 82% rename from plugins/incremental-ingestion-backend/package.json rename to plugins/catalog-backend-module-incremental-ingestion/package.json index f8ba9bf1f5..74ed0c8e5c 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -1,5 +1,5 @@ { - "name": "@backstage/plugin-incremental-ingestion-backend", + "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion", "description": "An entity provider for streaming large asset sources into the catalog", "version": "0.0.0", "main": "src/index.ts", @@ -7,24 +7,25 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, "backstage": { - "role": "backend-plugin" + "role": "backend-plugin-module" }, "homepage": "https://backstage.io", "repository": { "type": "git", "url": "https://github.com/backstage/backstage", - "directory": "plugins/incremental-ingestion-backend" + "directory": "plugins/catalog-backend-module-incremental-ingestion" }, "keywords": [ "backstage" ], "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "clean": "backstage-cli package clean", @@ -46,6 +47,7 @@ "express": "^4.17.1", "express-promise-router": "^4.1.0", "knex": "^2.0.0", + "lodash": "^4.17.21", "luxon": "^3.0.0", "uuid": "^8.3.2", "winston": "^3.2.1" @@ -54,6 +56,7 @@ "@backstage/cli": "workspace:^" }, "files": [ + "alpha", "dist", "migrations/**/*.{js,d.ts}" ] diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts similarity index 99% rename from plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts index 7f4ac4eb7c..3c91b24e2c 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts @@ -19,14 +19,14 @@ import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { Duration } from 'luxon'; import { v4 } from 'uuid'; +import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION } from '../types'; import { - INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IngestionRecord, IngestionRecordUpdate, IngestionUpsert, MarkRecord, MarkRecordInsert, -} from '../types'; +} from './tables'; export class IncrementalIngestionDatabaseManager { private client: Knex; diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts similarity index 93% rename from plugins/incremental-ingestion-backend/src/database/migrations.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts index 58ccd7c8c5..940a82107b 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts @@ -20,7 +20,7 @@ import { DB_MIGRATIONS_TABLE } from './tables'; export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( - '@backstage/plugin-incremental-ingestion-backend', + '@backstage/plugin-catalog-backend-module-incremental-ingestion', 'migrations', ); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts new file mode 100644 index 0000000000..00d40fb4a1 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; + +/** + * The shape of data inserted into or updated in the `ingestions` table. + */ +export interface IngestionUpsert { + /** + * The ingestion record id. + */ + id?: string; + /** + * The next action the incremental entity provider will take. + */ + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + /** + * Current status of the incremental entity provider. + */ + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; + /** + * The name of the incremental entity provider being updated. + */ + provider_name: string; + /** + * Date/time stamp for when the next action will trigger. + */ + next_action_at?: Date; + /** + * A record of the last error generated by the incremental entity provider. + */ + last_error?: string | null; + /** + * The number of attempts the provider has attempted during the current cycle. + */ + attempts?: number; + /** + * Date/time stamp for the completion of ingestion. + */ + ingestion_completed_at?: Date | string | null; + /** + * Date/time stamp for the end of the rest cycle before the next ingestion. + */ + rest_completed_at?: Date | string | null; + /** + * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. + */ + completion_ticket: string; +} + +/** + * This interface is for updating an existing ingestion record. + */ +export interface IngestionRecordUpdate { + ingestionId: string; + update: Partial; +} + +/** + * The expected response from the `ingestion_marks` table. + */ +export interface MarkRecord { + id: string; + sequence: number; + ingestion_id: string; + cursor: string; + created_at: string; +} + +/** + * The expected response from the `ingestions` table. + */ +export interface IngestionRecord extends IngestionUpsert { + id: string; + next_action_at: Date; + /** + * The date/time the ingestion record was created. + */ + created_at: string; +} + +/** + * This interface supplies all the values for adding an ingestion mark. + */ +export interface MarkRecordInsert { + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts similarity index 99% rename from plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 922f181503..07768c08b7 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -22,7 +22,6 @@ import { } from '../types'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import type { AbortSignal } from 'node-abort-controller'; - import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/index.ts similarity index 79% rename from plugins/incremental-ingestion-backend/src/index.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/index.ts index 7d661c2b98..84d5fcf741 100644 --- a/plugins/incremental-ingestion-backend/src/index.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/index.ts @@ -20,11 +20,12 @@ * @packageDocumentation */ -export { IncrementalCatalogBuilder } from './service/IncrementalCatalogBuilder'; -export type { - EntityIteratorResult, - IncrementalEntityProvider, - IncrementalEntityProviderOptions, +export * from './module'; +export * from './service'; +export { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, - PluginEnvironment, + type EntityIteratorResult, + type IncrementalEntityProvider, + type IncrementalEntityProviderOptions, + type PluginEnvironment, } from './types'; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts new file mode 100644 index 0000000000..986a8fbf62 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { Logger, loggerToWinstonLogger } from '@backstage/backend-plugin-api'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { stringifyError } from '@backstage/errors'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import once from 'lodash/once'; +import { Duration } from 'luxon'; +import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { Deferred } from '../util'; + +const applyDatabaseMigrationsOnce = once(applyDatabaseMigrations); + +/** + * Helps in the creation of the catalog entity providers that wrap the + * incremental ones. + */ +export class WrapperProviders { + private numberOfProvidersToConnect = 0; + private readonly readySignal = new Deferred(); + + constructor( + private readonly deps: { + config: Config; + logger: Logger; + database: PluginDatabaseManager; + scheduler: PluginTaskScheduler; + }, + ) {} + + wrap( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ): EntityProvider { + this.numberOfProvidersToConnect += 1; + + const wrapper: EntityProvider = { + getProviderName: () => provider.getProviderName(), + connect: async connection => { + this.readySignal.then(() => + this.startProvider(provider, options, connection), + ); + + this.numberOfProvidersToConnect -= 1; + + if (this.numberOfProvidersToConnect === 0) { + this.readySignal.resolve(); + } + }, + }; + + return wrapper; + } + + private async startProvider( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + connection: EntityProviderConnection, + ) { + const logger = loggerToWinstonLogger( + this.deps.logger.child({ + entityProvider: provider.getProviderName(), + }), + ); + + try { + const client = await this.deps.database.getClient(); + await applyDatabaseMigrationsOnce(client); + + const { burstInterval, burstLength, restLength } = options; + + logger.info(`Connecting`); + + const manager = new IncrementalIngestionDatabaseManager({ + client, + }); + const engine = new IncrementalIngestionEngine({ + ...options, + ready: this.readySignal, + manager, + logger, + provider, + restLength, + connection, + }); + + const frequency = Duration.isDuration(burstInterval) + ? burstInterval + : Duration.fromObject(burstInterval); + const length = Duration.isDuration(burstLength) + ? burstLength + : Duration.fromObject(burstLength); + + await this.deps.scheduler.scheduleTask({ + id: provider.getProviderName(), + fn: engine.taskFn.bind(engine), + frequency, + timeout: length, + }); + } catch (error) { + logger.warn( + `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError( + error, + )}`, + ); + throw error; + } + } +} diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..80ac169f21 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + databaseServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { WrapperProviders } from './WrapperProviders'; + +/** + * Registers the incremental entity provider with the catalog processing extension point. + * + * @alpha + */ +export const incrementalIngestionEntityProviderCatalogModule = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalIngestionEntityProvider', + register( + env, + options: { + providers: Array<{ + provider: IncrementalEntityProvider; + options: IncrementalEntityProviderOptions; + }>; + }, + ) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + database: databaseServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, ...otherDeps }) { + const providers = new WrapperProviders(otherDeps); + for (const entry of options.providers) { + catalog.addEntityProvider( + providers.wrap(entry.provider, entry.options), + ); + } + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts new file mode 100644 index 0000000000..577cf193e2 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { incrementalIngestionEntityProviderCatalogModule } from './incrementalIngestionEntityProviderCatalogModule'; diff --git a/plugins/incremental-ingestion-backend/src/router/paths.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts similarity index 100% rename from plugins/incremental-ingestion-backend/src/router/paths.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts diff --git a/plugins/incremental-ingestion-backend/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts similarity index 100% rename from plugins/incremental-ingestion-backend/src/router/routes.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts similarity index 85% rename from plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index 680c76011e..d6bc5549aa 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -25,35 +25,7 @@ import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { createIncrementalProviderRouter } from '../router/routes'; - -class Deferred implements Promise { - #resolve?: (value: T) => void; - #reject?: (error: Error) => void; - - get resolve() { - return this.#resolve!; - } - get reject() { - return this.#reject!; - } - - then: Promise['then']; - catch: Promise['catch']; - finally: Promise['finally']; - - constructor() { - const promise = new Promise((resolve, reject) => { - this.#resolve = resolve; - this.#reject = reject; - }); - - this.then = promise.then.bind(promise); - this.catch = promise.catch.bind(promise); - this.finally = promise.finally.bind(promise); - } - - [Symbol.toStringTag]: 'Deferred' = 'Deferred'; -} +import { Deferred } from '../util'; /** @public */ export class IncrementalCatalogBuilder { diff --git a/plugins/incremental-ingestion-backend/src/database/tables.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts similarity index 82% rename from plugins/incremental-ingestion-backend/src/database/tables.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts index 6eee7e6dc5..5df92feed5 100644 --- a/plugins/incremental-ingestion-backend/src/database/tables.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; +export { IncrementalCatalogBuilder } from './IncrementalCatalogBuilder'; diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts similarity index 67% rename from plugins/incremental-ingestion-backend/src/types.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 47314626ed..c2603eea4e 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import type { PluginDatabaseManager, UrlReader, @@ -154,104 +155,3 @@ export interface IterationEngineOptions { ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; } - -/** - * The shape of data inserted into or updated in the `ingestions` table. - */ -export interface IngestionUpsert { - /** - * The ingestion record id. - */ - id?: string; - /** - * The next action the incremental entity provider will take. - */ - next_action: - | 'rest' - | 'ingest' - | 'backoff' - | 'cancel' - | 'nothing (done)' - | 'nothing (canceled)'; - /** - * Current status of the incremental entity provider. - */ - status: - | 'complete' - | 'bursting' - | 'resting' - | 'canceling' - | 'interstitial' - | 'backing off'; - /** - * The name of the incremental entity provider being updated. - */ - provider_name: string; - /** - * Date/time stamp for when the next action will trigger. - */ - next_action_at?: Date; - /** - * A record of the last error generated by the incremental entity provider. - */ - last_error?: string | null; - /** - * The number of attempts the provider has attempted during the current cycle. - */ - attempts?: number; - /** - * Date/time stamp for the completion of ingestion. - */ - ingestion_completed_at?: Date | string | null; - /** - * Date/time stamp for the end of the rest cycle before the next ingestion. - */ - rest_completed_at?: Date | string | null; - /** - * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. - */ - completion_ticket: string; -} - -/** - * This interface is for updating an existing ingestion record. - */ -export interface IngestionRecordUpdate { - ingestionId: string; - update: Partial; -} - -/** - * The expected response from the `ingestion_marks` table. - */ -export interface MarkRecord { - id: string; - sequence: number; - ingestion_id: string; - cursor: string; - created_at: string; -} - -/** - * The expected response from the `ingestions` table. - */ -export interface IngestionRecord extends IngestionUpsert { - id: string; - next_action_at: Date; - /** - * The date/time the ingestion record was created. - */ - created_at: string; -} - -/** - * This interface supplies all the values for adding an ingestion mark. - */ -export interface MarkRecordInsert { - record: { - id: string; - ingestion_id: string; - cursor: unknown; - sequence: number; - }; -} diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/util.ts b/plugins/catalog-backend-module-incremental-ingestion/src/util.ts new file mode 100644 index 0000000000..d9e820f826 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/util.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class Deferred implements Promise { + #resolve?: (value: T) => void; + #reject?: (error: Error) => void; + + public get resolve() { + return this.#resolve!; + } + public get reject() { + return this.#reject!; + } + + public then: Promise['then']; + public catch: Promise['catch']; + public finally: Promise['finally']; + + public constructor() { + const promise = new Promise((resolve, reject) => { + this.#resolve = resolve; + this.#reject = reject; + }); + + this.then = promise.then.bind(promise); + this.catch = promise.catch.bind(promise); + this.finally = promise.finally.bind(promise); + } + + [Symbol.toStringTag]: 'Deferred' = 'Deferred'; +} diff --git a/yarn.lock b/yarn.lock index 14274d6639..43cb63fa3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4619,6 +4619,32 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@types/express": ^4.17.6 + "@types/luxon": ^3.0.0 + express: ^4.17.1 + express-promise-router: ^4.1.0 + knex: ^2.0.0 + lodash: ^4.17.21 + luxon: ^3.0.0 + uuid: ^8.3.2 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend-module-ldap@workspace:plugins/catalog-backend-module-ldap": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-ldap@workspace:plugins/catalog-backend-module-ldap" @@ -6023,31 +6049,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": - version: 0.0.0-use.local - resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" - dependencies: - "@backstage/backend-common": "workspace:^" - "@backstage/backend-plugin-api": "workspace:^" - "@backstage/backend-tasks": "workspace:^" - "@backstage/catalog-model": "workspace:^" - "@backstage/cli": "workspace:^" - "@backstage/config": "workspace:^" - "@backstage/errors": "workspace:^" - "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-catalog-node": "workspace:^" - "@backstage/plugin-permission-common": "workspace:^" - "@types/express": ^4.17.6 - "@types/luxon": ^3.0.0 - express: ^4.17.1 - express-promise-router: ^4.1.0 - knex: ^2.0.0 - luxon: ^3.0.0 - uuid: ^8.3.2 - winston: ^3.2.1 - languageName: unknown - linkType: soft - "@backstage/plugin-jenkins-backend@workspace:^, @backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend"